From ecbca97ca47eb6c979f5f904b929efcdc613b83a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 13:55:05 +0800 Subject: [PATCH 01/34] more code --- source/dnode/vnode/src/inc/tsdb.h | 7 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 132 ++++++++++++++++++ 2 files changed, 136 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 7546b0943e..98a3ee9fdb 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -633,9 +633,10 @@ typedef struct SMergeTree { struct SLDataIter *pIter; } SMergeTree; -int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange); -void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter); -bool tMergeTreeNext(SMergeTree *pMTree); +int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t uid, + STimeWindow *pTimeWindow, SVersionRange *pVerRange); +void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter); +bool tMergeTreeNext(SMergeTree *pMTree); TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index dbaf9b234c..a969a3c080 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1608,3 +1608,135 @@ _err: tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } + +// =============== PAGE-WISE FILE =============== +typedef struct { + TdFilePtr pFD; + int32_t szPage; + int32_t nBuf; + uint8_t *pBuf; + int64_t pgno; +} STsdbFD; + +int32_t tsdbOpenFile(const char *path, int32_t opt, STsdbFD *pFD) { + int32_t code = 0; + + pFD->pFD = taosOpenFile(path, opt); + if (pFD->pFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + + pFD->szPage = 4096; + pFD->nBuf = 0; + pFD->pBuf = taosMemoryMalloc(pFD->szPage); + if (pFD->pBuf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + +_exit: + return code; +} + +void tsdbCloseFile(STsdbFD *pFD) { + taosMemoryFree(pFD->pBuf); + taosCloseFile(&pFD->pFD); +} + +int32_t tsdbSyncFile(STsdbFD *pFD) { + int32_t code = 0; + + if (taosFsyncFile(pFD->pFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + +_exit: + return code; +} + +int32_t tsdbWriteFile(STsdbFD *pFD, uint8_t *pBuf, int32_t nBuf) { + int32_t code = 0; + + int32_t n = 0; + while (n < nBuf) { + int32_t remain = pFD->szPage - pFD->nBuf - sizeof(TSCKSUM); + int32_t size = TMIN(remain, nBuf - n); + + memcpy(pFD->pBuf + pFD->nBuf, pBuf + n, size); + n += size; + pFD->nBuf += size; + + if (pFD->nBuf + sizeof(TSCKSUM) == pFD->szPage) { + taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); + + int64_t n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + + pFD->nBuf = 0; + } + } + +_exit: + return code; +} + +static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { + int32_t code = 0; + + int64_t n = taosLSeekFile(pFD->pFD, pgno * pFD->szPage, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + + n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } else if (n < pFD->szPage) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + + if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + + pFD->pgno = pgno; + +_exit: + return code; +} + +int64_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t count) { + int32_t code = 0; + + int64_t pgno = offset / pFD->szPage; + int64_t n = 0; + if (pFD->pgno == pgno) { + int64_t bOff = offset % pFD->szPage; + int64_t nRead = TMIN(pFD->szPage - bOff - sizeof(TSCKSUM), count); + memcpy(pBuf + n, pFD->pBuf + bOff, nRead); + n = nRead; + } + + while (n < count) { + code = tsdbReadFilePage(pFD, pgno); + if (code) goto _exit; + + pgno++; + + int64_t nRead = TMIN(pFD->szPage - sizeof(TSCKSUM), count - n); + memcpy(pBuf + n, pFD->pBuf, nRead); + n += nRead; + } + +_exit: + return code; +} \ No newline at end of file From 687dd81c448bccc00c1a99b9f9727678a4c3b889 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 14:26:57 +0800 Subject: [PATCH 02/34] refact more code --- source/dnode/vnode/src/inc/tsdb.h | 16 ++++++++-------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 8 ++++---- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 9 +++++---- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 98a3ee9fdb..33c2c8a737 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -66,11 +66,11 @@ typedef struct SSmaInfo SSmaInfo; typedef struct SBlockCol SBlockCol; typedef struct SVersionRange SVersionRange; -#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) -#define TSDB_MAX_SUBBLOCKS 8 -#define TSDB_MAX_LAST_FILE 16 -#define TSDB_DEFAULT_LAST_FILE 8 -#define TSDB_FHDR_SIZE 512 +#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) +#define TSDB_MAX_SUBBLOCKS 8 +#define TSDB_MAX_SST_FILE 16 +#define TSDB_DEFAULT_SST_FILE 8 +#define TSDB_FHDR_SIZE 512 #define HAS_NONE ((int8_t)0x1) #define HAS_NULL ((int8_t)0x2) @@ -563,7 +563,7 @@ struct SDFileSet { SDataFile *pDataF; SSmaFile *pSmaF; uint8_t nSstF; - SSstFile *aSstF[TSDB_MAX_LAST_FILE]; + SSstFile *aSstF[TSDB_MAX_SST_FILE]; }; struct SRowIter { @@ -598,7 +598,7 @@ struct SDataFWriter { SHeadFile fHead; SDataFile fData; SSmaFile fSma; - SSstFile fSst[TSDB_MAX_LAST_FILE]; + SSstFile fSst[TSDB_MAX_SST_FILE]; uint8_t *aBuf[4]; }; @@ -615,7 +615,7 @@ struct SDataFReader { TdFilePtr pHeadFD; TdFilePtr pDataFD; TdFilePtr pSmaFD; - TdFilePtr aLastFD[TSDB_MAX_LAST_FILE]; + TdFilePtr aLastFD[TSDB_MAX_SST_FILE]; uint8_t *aBuf[3]; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 42ab6ce3b2..5b827d8ed2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -71,7 +71,7 @@ typedef struct { SDataIter *pIter; SRBTree rbt; SDataIter dataIter; - SDataIter aDataIter[TSDB_MAX_LAST_FILE]; + SDataIter aDataIter[TSDB_MAX_SST_FILE]; int8_t toLastOnly; }; struct { @@ -760,7 +760,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; - pCommitter->maxLast = TSDB_DEFAULT_LAST_FILE; // TODO: make it as a config + pCommitter->maxLast = TSDB_DEFAULT_SST_FILE; // TODO: make it as a config pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); if (pCommitter->aTbDataP == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -790,7 +790,7 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { if (code) goto _exit; // merger - for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) { + for (int32_t iSst = 0; iSst < TSDB_MAX_SST_FILE; iSst++) { SDataIter *pIter = &pCommitter->aDataIter[iSst]; pIter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk)); if (pIter->aSstBlk == NULL) { @@ -832,7 +832,7 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { tBlockDataDestroy(&pCommitter->dReader.bData, 1); // merger - for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) { + for (int32_t iSst = 0; iSst < TSDB_MAX_SST_FILE; iSst++) { SDataIter *pIter = &pCommitter->aDataIter[iSst]; taosArrayDestroy(pIter->aSstBlk); tBlockDataDestroy(&pIter->bData, 1); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index ce36d74467..0589199d24 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -273,7 +273,8 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { } } -int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange) { +int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t uid, + STimeWindow *pTimeWindow, SVersionRange *pVerRange) { pMTree->backward = backward; pMTree->pIter = NULL; pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); @@ -284,8 +285,8 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFRead tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); int32_t code = TSDB_CODE_OUT_OF_MEMORY; - struct SLDataIter* pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; - for(int32_t i = 0; i < pFReader->pSet->nSstF; ++i) { // open all last file + struct SLDataIter *pIterList[TSDB_DEFAULT_SST_FILE] = {0}; + for (int32_t i = 0; i < pFReader->pSet->nSstF; ++i) { // open all last file code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); if (code != TSDB_CODE_SUCCESS) { goto _end; @@ -302,7 +303,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFRead return code; - _end: +_end: tMergeTreeClose(pMTree); return code; } From f963f5bfccd67534c7d633d51856052a7d5d89fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 17:02:13 +0800 Subject: [PATCH 03/34] more code --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index a969a3c080..d02d970943 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1628,6 +1628,7 @@ int32_t tsdbOpenFile(const char *path, int32_t opt, STsdbFD *pFD) { } pFD->szPage = 4096; + pFD->pgno = 0; pFD->nBuf = 0; pFD->pBuf = taosMemoryMalloc(pFD->szPage); if (pFD->pBuf == NULL) { @@ -1656,7 +1657,7 @@ _exit: return code; } -int32_t tsdbWriteFile(STsdbFD *pFD, uint8_t *pBuf, int32_t nBuf) { +int32_t tsdbWriteFile(STsdbFD *pFD, uint8_t *pBuf, int32_t nBuf, int64_t *offset) { int32_t code = 0; int32_t n = 0; From 7fe743d00fc65eb0d5dabefed9528edbceb125c9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 17:04:49 +0800 Subject: [PATCH 04/34] refact code for further change --- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 879 +++++++++--------- 1 file changed, 439 insertions(+), 440 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index d02d970943..3aa3aa5182 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -14,382 +14,136 @@ */ #include "tsdb.h" +// =============== PAGE-WISE FILE =============== +typedef struct { + TdFilePtr pFD; + int32_t szPage; + int32_t nBuf; + uint8_t *pBuf; + int64_t pgno; +} STsdbFD; -// SDelFWriter ==================================================== -int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { - int32_t code = 0; - char fname[TSDB_FILENAME_LEN]; - char hdr[TSDB_FHDR_SIZE] = {0}; - SDelFWriter *pDelFWriter; - int64_t n; - - // alloc - pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter)); - if (pDelFWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - pDelFWriter->pTsdb = pTsdb; - pDelFWriter->fDel = *pFile; - - tsdbDelFileName(pTsdb, pFile, fname); - pDelFWriter->pWriteH = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE); - if (pDelFWriter->pWriteH == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // update header - n = taosWriteFile(pDelFWriter->pWriteH, &hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pDelFWriter->fDel.size = TSDB_FHDR_SIZE; - pDelFWriter->fDel.offset = 0; - - *ppWriter = pDelFWriter; - return code; - -_err: - tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppWriter = NULL; - return code; -} - -int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { - int32_t code = 0; - SDelFWriter *pWriter = *ppWriter; - STsdb *pTsdb = pWriter->pTsdb; - - // sync - if (sync && taosFsyncFile(pWriter->pWriteH) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // close - if (taosCloseFile(&pWriter->pWriteH) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree(pWriter->aBuf[iBuf]); - } - taosMemoryFree(pWriter); - - *ppWriter = NULL; - return code; - -_err: - tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) { +int32_t tsdbOpenFile(const char *path, int32_t opt, STsdbFD *pFD) { int32_t code = 0; - int64_t size; - int64_t n; - // prepare - size = sizeof(uint32_t); - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData)); - } - size += sizeof(TSCKSUM); - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData)); - } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); - - // write - n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); - if (n < 0) { + pFD->pFD = taosOpenFile(path, opt); + if (pFD->pFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + goto _exit; } - ASSERT(n == size); - - // update - pDelIdx->offset = pWriter->fDel.size; - pDelIdx->size = size; - pWriter->fDel.size += size; - - return code; - -_err: - tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { - int32_t code = 0; - int64_t size; - int64_t n; - SDelIdx *pDelIdx; - - // prepare - size = sizeof(uint32_t); - for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx)); - } - size += sizeof(TSCKSUM); - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx)); - } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); - - // write - n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // update - pWriter->fDel.offset = pWriter->fDel.size; - pWriter->fDel.size += size; - - return code; - -_err: - tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) { - int32_t code = 0; - char hdr[TSDB_FHDR_SIZE]; - int64_t size = TSDB_FHDR_SIZE; - int64_t n; - - // build - memset(hdr, 0, size); - tPutDelFile(hdr, &pWriter->fDel); - taosCalcChecksumAppend(0, hdr, size); - - // seek - if (taosLSeekFile(pWriter->pWriteH, 0, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // write - n = taosWriteFile(pWriter->pWriteH, hdr, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - return code; - -_err: - tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -// SDelFReader ==================================================== -struct SDelFReader { - STsdb *pTsdb; - SDelFile fDel; - TdFilePtr pReadH; - - uint8_t *aBuf[1]; -}; - -int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { - int32_t code = 0; - char fname[TSDB_FILENAME_LEN]; - SDelFReader *pDelFReader; - int64_t n; - - // alloc - pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader)); - if (pDelFReader == NULL) { + pFD->szPage = 4096; + pFD->pgno = 0; + pFD->nBuf = 0; + pFD->pBuf = taosMemoryMalloc(pFD->szPage); + if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + goto _exit; } - // open impl - pDelFReader->pTsdb = pTsdb; - pDelFReader->fDel = *pFile; - - tsdbDelFileName(pTsdb, pFile, fname); - pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ); - if (pDelFReader->pReadH == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - taosMemoryFree(pDelFReader); - goto _err; - } - -_exit: - *ppReader = pDelFReader; - return code; - -_err: - tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppReader = NULL; - return code; -} - -int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { - int32_t code = 0; - SDelFReader *pReader = *ppReader; - - if (pReader) { - if (taosCloseFile(&pReader->pReadH) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } - for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree(pReader->aBuf[iBuf]); - } - taosMemoryFree(pReader); - } - *ppReader = NULL; - _exit: return code; } -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { +void tsdbCloseFile(STsdbFD *pFD) { + taosMemoryFree(pFD->pBuf); + taosCloseFile(&pFD->pFD); +} + +int32_t tsdbSyncFile(STsdbFD *pFD) { int32_t code = 0; - int64_t offset = pDelIdx->offset; - int64_t size = pDelIdx->size; - int64_t n; - taosArrayClear(aDelData); - - // seek - if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { + if (taosFsyncFile(pFD->pFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + goto _exit; } - // alloc - code = tRealloc(&pReader->aBuf[0], size); - if (code) goto _err; - - // read - n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // // decode - n = 0; - - uint32_t delimiter; - n += tGetU32(pReader->aBuf[0] + n, &delimiter); - while (n < size - sizeof(TSCKSUM)) { - SDelData delData; - n += tGetDelData(pReader->aBuf[0] + n, &delData); - - if (taosArrayPush(aDelData, &delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - - ASSERT(n == size - sizeof(TSCKSUM)); - - return code; - -_err: - tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); +_exit: return code; } -int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { +int32_t tsdbWriteFile(STsdbFD *pFD, uint8_t *pBuf, int32_t nBuf, int64_t *offset) { int32_t code = 0; - int32_t n; - int64_t offset = pReader->fDel.offset; - int64_t size = pReader->fDel.size - offset; - taosArrayClear(aDelIdx); + int32_t n = 0; + while (n < nBuf) { + int32_t remain = pFD->szPage - pFD->nBuf - sizeof(TSCKSUM); + int32_t size = TMIN(remain, nBuf - n); - // seek - if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + memcpy(pFD->pBuf + pFD->nBuf, pBuf + n, size); + n += size; + pFD->nBuf += size; - // alloc - code = tRealloc(&pReader->aBuf[0], size); - if (code) goto _err; + if (pFD->nBuf + sizeof(TSCKSUM) == pFD->szPage) { + taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); - // read - n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + int64_t n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // decode - n = 0; - uint32_t delimiter; - n += tGetU32(pReader->aBuf[0] + n, &delimiter); - ASSERT(delimiter == TSDB_FILE_DLMT); - - while (n < size - sizeof(TSCKSUM)) { - SDelIdx delIdx; - - n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx); - - if (taosArrayPush(aDelIdx, &delIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + pFD->nBuf = 0; } } - ASSERT(n == size - sizeof(TSCKSUM)); - +_exit: return code; +} -_err: - tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); +static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { + int32_t code = 0; + + int64_t n = taosLSeekFile(pFD->pFD, pgno * pFD->szPage, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + + n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } else if (n < pFD->szPage) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + + if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + + pFD->pgno = pgno; + +_exit: + return code; +} + +int64_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t count) { + int32_t code = 0; + + int64_t pgno = offset / pFD->szPage; + int64_t n = 0; + if (pFD->pgno == pgno) { + int64_t bOff = offset % pFD->szPage; + int64_t nRead = TMIN(pFD->szPage - bOff - sizeof(TSCKSUM), count); + memcpy(pBuf + n, pFD->pBuf + bOff, nRead); + n = nRead; + } + + while (n < count) { + code = tsdbReadFilePage(pFD, pgno); + if (code) goto _exit; + + pgno++; + + int64_t nRead = TMIN(pFD->szPage - sizeof(TSCKSUM), count - n); + memcpy(pBuf + n, pFD->pBuf, nRead); + n += nRead; + } + +_exit: return code; } @@ -1609,135 +1363,380 @@ _err: return code; } -// =============== PAGE-WISE FILE =============== -typedef struct { - TdFilePtr pFD; - int32_t szPage; - int32_t nBuf; - uint8_t *pBuf; - int64_t pgno; -} STsdbFD; +// SDelFWriter ==================================================== +int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { + int32_t code = 0; + char fname[TSDB_FILENAME_LEN]; + char hdr[TSDB_FHDR_SIZE] = {0}; + SDelFWriter *pDelFWriter; + int64_t n; -int32_t tsdbOpenFile(const char *path, int32_t opt, STsdbFD *pFD) { - int32_t code = 0; - - pFD->pFD = taosOpenFile(path, opt); - if (pFD->pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } - - pFD->szPage = 4096; - pFD->pgno = 0; - pFD->nBuf = 0; - pFD->pBuf = taosMemoryMalloc(pFD->szPage); - if (pFD->pBuf == NULL) { + // alloc + pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter)); + if (pDelFWriter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + goto _err; } + pDelFWriter->pTsdb = pTsdb; + pDelFWriter->fDel = *pFile; -_exit: - return code; -} - -void tsdbCloseFile(STsdbFD *pFD) { - taosMemoryFree(pFD->pBuf); - taosCloseFile(&pFD->pFD); -} - -int32_t tsdbSyncFile(STsdbFD *pFD) { - int32_t code = 0; - - if (taosFsyncFile(pFD->pFD) < 0) { + tsdbDelFileName(pTsdb, pFile, fname); + pDelFWriter->pWriteH = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE); + if (pDelFWriter->pWriteH == NULL) { code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + goto _err; } + // update header + n = taosWriteFile(pDelFWriter->pWriteH, &hdr, TSDB_FHDR_SIZE); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pDelFWriter->fDel.size = TSDB_FHDR_SIZE; + pDelFWriter->fDel.offset = 0; + + *ppWriter = pDelFWriter; + return code; + +_err: + tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + *ppWriter = NULL; + return code; +} + +int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { + int32_t code = 0; + SDelFWriter *pWriter = *ppWriter; + STsdb *pTsdb = pWriter->pTsdb; + + // sync + if (sync && taosFsyncFile(pWriter->pWriteH) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // close + if (taosCloseFile(&pWriter->pWriteH) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree(pWriter->aBuf[iBuf]); + } + taosMemoryFree(pWriter); + + *ppWriter = NULL; + return code; + +_err: + tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) { + int32_t code = 0; + int64_t size; + int64_t n; + + // prepare + size = sizeof(uint32_t); + for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { + size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData)); + } + size += sizeof(TSCKSUM); + + // alloc + code = tRealloc(&pWriter->aBuf[0], size); + if (code) goto _err; + + // build + n = 0; + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); + for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { + n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData)); + } + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); + + ASSERT(n + sizeof(TSCKSUM) == size); + + // write + n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + ASSERT(n == size); + + // update + pDelIdx->offset = pWriter->fDel.size; + pDelIdx->size = size; + pWriter->fDel.size += size; + + return code; + +_err: + tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { + int32_t code = 0; + int64_t size; + int64_t n; + SDelIdx *pDelIdx; + + // prepare + size = sizeof(uint32_t); + for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { + size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx)); + } + size += sizeof(TSCKSUM); + + // alloc + code = tRealloc(&pWriter->aBuf[0], size); + if (code) goto _err; + + // build + n = 0; + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); + for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { + n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx)); + } + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); + + ASSERT(n + sizeof(TSCKSUM) == size); + + // write + n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // update + pWriter->fDel.offset = pWriter->fDel.size; + pWriter->fDel.size += size; + + return code; + +_err: + tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) { + int32_t code = 0; + char hdr[TSDB_FHDR_SIZE]; + int64_t size = TSDB_FHDR_SIZE; + int64_t n; + + // build + memset(hdr, 0, size); + tPutDelFile(hdr, &pWriter->fDel); + taosCalcChecksumAppend(0, hdr, size); + + // seek + if (taosLSeekFile(pWriter->pWriteH, 0, SEEK_SET) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // write + n = taosWriteFile(pWriter->pWriteH, hdr, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + return code; + +_err: + tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +// SDelFReader ==================================================== +struct SDelFReader { + STsdb *pTsdb; + SDelFile fDel; + TdFilePtr pReadH; + + uint8_t *aBuf[1]; +}; + +int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { + int32_t code = 0; + char fname[TSDB_FILENAME_LEN]; + SDelFReader *pDelFReader; + int64_t n; + + // alloc + pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader)); + if (pDelFReader == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + // open impl + pDelFReader->pTsdb = pTsdb; + pDelFReader->fDel = *pFile; + + tsdbDelFileName(pTsdb, pFile, fname); + pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ); + if (pDelFReader->pReadH == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pDelFReader); + goto _err; + } + +_exit: + *ppReader = pDelFReader; + return code; + +_err: + tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + *ppReader = NULL; + return code; +} + +int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { + int32_t code = 0; + SDelFReader *pReader = *ppReader; + + if (pReader) { + if (taosCloseFile(&pReader->pReadH) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree(pReader->aBuf[iBuf]); + } + taosMemoryFree(pReader); + } + *ppReader = NULL; + _exit: return code; } -int32_t tsdbWriteFile(STsdbFD *pFD, uint8_t *pBuf, int32_t nBuf, int64_t *offset) { +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { int32_t code = 0; + int64_t offset = pDelIdx->offset; + int64_t size = pDelIdx->size; + int64_t n; - int32_t n = 0; - while (n < nBuf) { - int32_t remain = pFD->szPage - pFD->nBuf - sizeof(TSCKSUM); - int32_t size = TMIN(remain, nBuf - n); + taosArrayClear(aDelData); - memcpy(pFD->pBuf + pFD->nBuf, pBuf + n, size); - n += size; - pFD->nBuf += size; + // seek + if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } - if (pFD->nBuf + sizeof(TSCKSUM) == pFD->szPage) { - taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); + // alloc + code = tRealloc(&pReader->aBuf[0], size); + if (code) goto _err; - int64_t n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } + // read + n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n < size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } - pFD->nBuf = 0; + // check + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // // decode + n = 0; + + uint32_t delimiter; + n += tGetU32(pReader->aBuf[0] + n, &delimiter); + while (n < size - sizeof(TSCKSUM)) { + SDelData delData; + n += tGetDelData(pReader->aBuf[0] + n, &delData); + + if (taosArrayPush(aDelData, &delData) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } } -_exit: + ASSERT(n == size - sizeof(TSCKSUM)); + + return code; + +_err: + tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } -static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { +int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { int32_t code = 0; + int32_t n; + int64_t offset = pReader->fDel.offset; + int64_t size = pReader->fDel.size - offset; - int64_t n = taosLSeekFile(pFD->pFD, pgno * pFD->szPage, SEEK_SET); + taosArrayClear(aDelIdx); + + // seek + if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // alloc + code = tRealloc(&pReader->aBuf[0], size); + if (code) goto _err; + + // read + n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } - - n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } else if (n < pFD->szPage) { + goto _err; + } else if (n < size) { code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; + goto _err; } - if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { + // check + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; + goto _err; } - pFD->pgno = pgno; + // decode + n = 0; + uint32_t delimiter; + n += tGetU32(pReader->aBuf[0] + n, &delimiter); + ASSERT(delimiter == TSDB_FILE_DLMT); -_exit: + while (n < size - sizeof(TSCKSUM)) { + SDelIdx delIdx; + + n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx); + + if (taosArrayPush(aDelIdx, &delIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + + ASSERT(n == size - sizeof(TSCKSUM)); + + return code; + +_err: + tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } - -int64_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t count) { - int32_t code = 0; - - int64_t pgno = offset / pFD->szPage; - int64_t n = 0; - if (pFD->pgno == pgno) { - int64_t bOff = offset % pFD->szPage; - int64_t nRead = TMIN(pFD->szPage - bOff - sizeof(TSCKSUM), count); - memcpy(pBuf + n, pFD->pBuf + bOff, nRead); - n = nRead; - } - - while (n < count) { - code = tsdbReadFilePage(pFD, pgno); - if (code) goto _exit; - - pgno++; - - int64_t nRead = TMIN(pFD->szPage - sizeof(TSCKSUM), count - n); - memcpy(pBuf + n, pFD->pBuf, nRead); - n += nRead; - } - -_exit: - return code; -} \ No newline at end of file From 8329284a455abe85d63bbcbaa69e1ba38a658709 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 17:15:45 +0800 Subject: [PATCH 05/34] refact for further dev --- source/dnode/vnode/src/inc/tsdb.h | 12 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 1355 ++++++++--------- 2 files changed, 671 insertions(+), 696 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 33c2c8a737..0e9ec7bc24 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -586,6 +586,12 @@ struct SDelFWriter { uint8_t *aBuf[1]; }; +struct STsdbReadSnap { + SMemTable *pMem; + SMemTable *pIMem; + STsdbFS fs; +}; + struct SDataFWriter { STsdb *pTsdb; SDFileSet wSet; @@ -603,12 +609,6 @@ struct SDataFWriter { uint8_t *aBuf[4]; }; -struct STsdbReadSnap { - SMemTable *pMem; - SMemTable *pIMem; - STsdbFS fs; -}; - struct SDataFReader { STsdb *pTsdb; SDFileSet *pSet; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 3aa3aa5182..4bbbf24c47 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -23,7 +23,7 @@ typedef struct { int64_t pgno; } STsdbFD; -int32_t tsdbOpenFile(const char *path, int32_t opt, STsdbFD *pFD) { +static int32_t tsdbOpenFile(const char *path, int32_t opt, STsdbFD *pFD) { int32_t code = 0; pFD->pFD = taosOpenFile(path, opt); @@ -45,12 +45,12 @@ _exit: return code; } -void tsdbCloseFile(STsdbFD *pFD) { +static void tsdbCloseFile(STsdbFD *pFD) { taosMemoryFree(pFD->pBuf); taosCloseFile(&pFD->pFD); } -int32_t tsdbSyncFile(STsdbFD *pFD) { +static int32_t tsdbSyncFile(STsdbFD *pFD) { int32_t code = 0; if (taosFsyncFile(pFD->pFD) < 0) { @@ -62,7 +62,7 @@ _exit: return code; } -int32_t tsdbWriteFile(STsdbFD *pFD, uint8_t *pBuf, int32_t nBuf, int64_t *offset) { +static int32_t tsdbWriteFile(STsdbFD *pFD, uint8_t *pBuf, int32_t nBuf, int64_t *offset) { int32_t code = 0; int32_t n = 0; @@ -120,7 +120,7 @@ _exit: return code; } -int64_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t count) { +static int64_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t count) { int32_t code = 0; int64_t pgno = offset / pFD->szPage; @@ -147,6 +147,666 @@ _exit: return code; } +// SDataFWriter ==================================================== +int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { + int32_t code = 0; + int32_t flag; + int64_t n; + SDataFWriter *pWriter = NULL; + char fname[TSDB_FILENAME_LEN]; + char hdr[TSDB_FHDR_SIZE] = {0}; + + // alloc + pWriter = taosMemoryCalloc(1, sizeof(*pWriter)); + if (pWriter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pWriter->pTsdb = pTsdb; + pWriter->wSet = (SDFileSet){ + .diskId = pSet->diskId, + .fid = pSet->fid, + .pHeadF = &pWriter->fHead, + .pDataF = &pWriter->fData, + .pSmaF = &pWriter->fSma, + .nSstF = pSet->nSstF // + }; + pWriter->fHead = *pSet->pHeadF; + pWriter->fData = *pSet->pDataF; + pWriter->fSma = *pSet->pSmaF; + for (int8_t iSst = 0; iSst < pSet->nSstF; iSst++) { + pWriter->wSet.aSstF[iSst] = &pWriter->fSst[iSst]; + pWriter->fSst[iSst] = *pSet->aSstF[iSst]; + } + + // head + flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname); + pWriter->pHeadFD = taosOpenFile(fname, flag); + if (pWriter->pHeadFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + ASSERT(n == TSDB_FHDR_SIZE); + + pWriter->fHead.size += TSDB_FHDR_SIZE; + + // data + if (pWriter->fData.size == 0) { + flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + } else { + flag = TD_FILE_WRITE; + } + tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname); + pWriter->pDataFD = taosOpenFile(fname, flag); + if (pWriter->pDataFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (pWriter->fData.size == 0) { + n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pWriter->fData.size += TSDB_FHDR_SIZE; + } else { + n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_END); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + ASSERT(n == pWriter->fData.size); + } + + // sma + if (pWriter->fSma.size == 0) { + flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + } else { + flag = TD_FILE_WRITE; + } + tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname); + pWriter->pSmaFD = taosOpenFile(fname, flag); + if (pWriter->pSmaFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (pWriter->fSma.size == 0) { + n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pWriter->fSma.size += TSDB_FHDR_SIZE; + } else { + n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_END); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + ASSERT(n == pWriter->fSma.size); + } + + // sst + ASSERT(pWriter->fSst[pSet->nSstF - 1].size == 0); + flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + tsdbSstFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSst[pSet->nSstF - 1], fname); + pWriter->pLastFD = taosOpenFile(fname, flag); + if (pWriter->pLastFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + pWriter->fSst[pWriter->wSet.nSstF - 1].size += TSDB_FHDR_SIZE; + + *ppWriter = pWriter; + return code; + +_err: + tsdbError("vgId:%d, tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + *ppWriter = NULL; + return code; +} + +int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { + int32_t code = 0; + STsdb *pTsdb = NULL; + + if (*ppWriter == NULL) goto _exit; + + pTsdb = (*ppWriter)->pTsdb; + if (sync) { + if (taosFsyncFile((*ppWriter)->pHeadFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosFsyncFile((*ppWriter)->pDataFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosFsyncFile((*ppWriter)->pSmaFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosFsyncFile((*ppWriter)->pLastFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } + + if (taosCloseFile(&(*ppWriter)->pHeadFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosCloseFile(&(*ppWriter)->pDataFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosCloseFile(&(*ppWriter)->pSmaFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosCloseFile(&(*ppWriter)->pLastFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree((*ppWriter)->aBuf[iBuf]); + } + taosMemoryFree(*ppWriter); +_exit: + *ppWriter = NULL; + return code; + +_err: + tsdbError("vgId:%d, data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { + int32_t code = 0; + int64_t n; + char hdr[TSDB_FHDR_SIZE]; + + // head ============== + memset(hdr, 0, TSDB_FHDR_SIZE); + tPutHeadFile(hdr, &pWriter->fHead); + taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); + + n = taosLSeekFile(pWriter->pHeadFD, 0, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // data ============== + memset(hdr, 0, TSDB_FHDR_SIZE); + tPutDataFile(hdr, &pWriter->fData); + taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); + + n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // sma ============== + memset(hdr, 0, TSDB_FHDR_SIZE); + tPutSmaFile(hdr, &pWriter->fSma); + taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); + + n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // sst ============== + memset(hdr, 0, TSDB_FHDR_SIZE); + tPutSstFile(hdr, &pWriter->fSst[pWriter->wSet.nSstF - 1]); + taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); + + n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + return code; + +_err: + tsdbError("vgId:%d, update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { + int32_t code = 0; + SHeadFile *pHeadFile = &pWriter->fHead; + int64_t size = 0; + int64_t n; + + // check + if (taosArrayGetSize(aBlockIdx) == 0) { + pHeadFile->offset = pHeadFile->size; + goto _exit; + } + + // prepare + size = sizeof(uint32_t); + for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { + size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx)); + } + size += sizeof(TSCKSUM); + + // alloc + code = tRealloc(&pWriter->aBuf[0], size); + if (code) goto _err; + + // build + n = 0; + n = tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); + for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { + n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx)); + } + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); + + ASSERT(n + sizeof(TSCKSUM) == size); + + // write + n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // update + pHeadFile->offset = pHeadFile->size; + pHeadFile->size += size; + +_exit: + tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", TD_VID(pWriter->pTsdb->pVnode), + pHeadFile->offset, size, taosArrayGetSize(aBlockIdx)); + return code; + +_err: + tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBlockIdx) { + int32_t code = 0; + SHeadFile *pHeadFile = &pWriter->fHead; + int64_t size; + int64_t n; + + ASSERT(mBlock->nItem > 0); + + // alloc + size = sizeof(uint32_t) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); + code = tRealloc(&pWriter->aBuf[0], size); + if (code) goto _err; + + // build + n = 0; + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); + n += tPutMapData(pWriter->aBuf[0] + n, mBlock); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); + + ASSERT(n + sizeof(TSCKSUM) == size); + + // write + n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // update + pBlockIdx->offset = pHeadFile->size; + pBlockIdx->size = size; + pHeadFile->size += size; + + tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 + " size:%" PRId64 " nItem:%d", + TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, + pBlockIdx->offset, pBlockIdx->size, mBlock->nItem); + return code; + +_err: + tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { + int32_t code = 0; + SSstFile *pSstFile = &pWriter->fSst[pWriter->wSet.nSstF - 1]; + int64_t size; + int64_t n; + + // check + if (taosArrayGetSize(aSstBlk) == 0) { + pSstFile->offset = pSstFile->size; + goto _exit; + } + + // size + size = sizeof(uint32_t); // TSDB_FILE_DLMT + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) { + size += tPutSstBlk(NULL, taosArrayGet(aSstBlk, iBlockL)); + } + size += sizeof(TSCKSUM); + + // alloc + code = tRealloc(&pWriter->aBuf[0], size); + if (code) goto _err; + + // encode + n = 0; + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) { + n += tPutSstBlk(pWriter->aBuf[0] + n, taosArrayGet(aSstBlk, iBlockL)); + } + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); + + ASSERT(n + sizeof(TSCKSUM) == size); + + // write + n = taosWriteFile(pWriter->pLastFD, pWriter->aBuf[0], size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // update + pSstFile->offset = pSstFile->size; + pSstFile->size += size; + +_exit: + tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), + pSstFile->offset, size); + return code; + +_err: + tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { + int32_t code = 0; + + pSmaInfo->offset = 0; + pSmaInfo->size = 0; + + // encode + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + + if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue; + + SColumnDataAgg sma; + tsdbCalcColDataSMA(pColData, &sma); + + code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); + if (code) goto _err; + pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma); + } + + // write + if (pSmaInfo->size) { + int32_t size = pSmaInfo->size + sizeof(TSCKSUM); + + code = tRealloc(&pWriter->aBuf[0], size); + if (code) goto _err; + + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); + + int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->aBuf[0], size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pSmaInfo->offset = pWriter->fSma.size; + pWriter->fSma.size += size; + } + + return code; + +_err: + tsdbError("vgId:%d tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, + int8_t cmprAlg, int8_t toLast) { + int32_t code = 0; + + ASSERT(pBlockData->nRow > 0); + + pBlkInfo->offset = toLast ? pWriter->fSst[pWriter->wSet.nSstF - 1].size : pWriter->fData.size; + pBlkInfo->szBlock = 0; + pBlkInfo->szKey = 0; + + int32_t aBufN[4] = {0}; + code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN); + if (code) goto _err; + + // write ================= + TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; + + pBlkInfo->szKey = aBufN[3] + aBufN[2]; + pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; + + int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], aBufN[3]); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosWriteFile(pFD, pWriter->aBuf[2], aBufN[2]); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (aBufN[1]) { + n = taosWriteFile(pFD, pWriter->aBuf[1], aBufN[1]); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } + + if (aBufN[0]) { + n = taosWriteFile(pFD, pWriter->aBuf[0], aBufN[0]); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } + + // update info + if (toLast) { + pWriter->fSst[pWriter->wSet.nSstF - 1].size += pBlkInfo->szBlock; + } else { + pWriter->fData.size += pBlkInfo->szBlock; + } + + // ================= SMA ==================== + if (pSmaInfo) { + code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo); + if (code) goto _err; + } + +_exit: + tsdbTrace("vgId:%d tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d", + TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset, + pBlkInfo->szBlock); + return code; + +_err: + tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { + int32_t code = 0; + int64_t n; + int64_t size; + TdFilePtr pOutFD = NULL; // TODO + TdFilePtr PInFD = NULL; // TODO + char fNameFrom[TSDB_FILENAME_LEN]; + char fNameTo[TSDB_FILENAME_LEN]; + + // head + tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); + tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); + + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pHeadF->size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); + + // data + tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); + tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); + + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pDataF->size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); + + // sst + tsdbSstFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSstF[0], fNameFrom); + tsdbSstFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSstF[0], fNameTo); + + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aSstF[0]->size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); + + // sma + tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); + tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); + + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pSmaF->size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); + + return code; + +_err: + tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + // SDataFReader ==================================================== int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; @@ -679,690 +1339,6 @@ _exit: return code; } -// SDataFWriter ==================================================== -int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { - int32_t code = 0; - int32_t flag; - int64_t n; - SDataFWriter *pWriter = NULL; - char fname[TSDB_FILENAME_LEN]; - char hdr[TSDB_FHDR_SIZE] = {0}; - - // alloc - pWriter = taosMemoryCalloc(1, sizeof(*pWriter)); - if (pWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - pWriter->pTsdb = pTsdb; - pWriter->wSet = (SDFileSet){ - .diskId = pSet->diskId, - .fid = pSet->fid, - .pHeadF = &pWriter->fHead, - .pDataF = &pWriter->fData, - .pSmaF = &pWriter->fSma, - .nSstF = pSet->nSstF // - }; - pWriter->fHead = *pSet->pHeadF; - pWriter->fData = *pSet->pDataF; - pWriter->fSma = *pSet->pSmaF; - for (int8_t iSst = 0; iSst < pSet->nSstF; iSst++) { - pWriter->wSet.aSstF[iSst] = &pWriter->fSst[iSst]; - pWriter->fSst[iSst] = *pSet->aSstF[iSst]; - } - - // head - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname); - pWriter->pHeadFD = taosOpenFile(fname, flag); - if (pWriter->pHeadFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - ASSERT(n == TSDB_FHDR_SIZE); - - pWriter->fHead.size += TSDB_FHDR_SIZE; - - // data - if (pWriter->fData.size == 0) { - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_WRITE; - } - tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname); - pWriter->pDataFD = taosOpenFile(fname, flag); - if (pWriter->pDataFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - if (pWriter->fData.size == 0) { - n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pWriter->fData.size += TSDB_FHDR_SIZE; - } else { - n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_END); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - ASSERT(n == pWriter->fData.size); - } - - // sma - if (pWriter->fSma.size == 0) { - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_WRITE; - } - tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname); - pWriter->pSmaFD = taosOpenFile(fname, flag); - if (pWriter->pSmaFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - if (pWriter->fSma.size == 0) { - n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pWriter->fSma.size += TSDB_FHDR_SIZE; - } else { - n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_END); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - ASSERT(n == pWriter->fSma.size); - } - - // sst - ASSERT(pWriter->fSst[pSet->nSstF - 1].size == 0); - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - tsdbSstFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSst[pSet->nSstF - 1], fname); - pWriter->pLastFD = taosOpenFile(fname, flag); - if (pWriter->pLastFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - pWriter->fSst[pWriter->wSet.nSstF - 1].size += TSDB_FHDR_SIZE; - - *ppWriter = pWriter; - return code; - -_err: - tsdbError("vgId:%d, tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppWriter = NULL; - return code; -} - -int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { - int32_t code = 0; - STsdb *pTsdb = NULL; - - if (*ppWriter == NULL) goto _exit; - - pTsdb = (*ppWriter)->pTsdb; - if (sync) { - if (taosFsyncFile((*ppWriter)->pHeadFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosFsyncFile((*ppWriter)->pDataFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosFsyncFile((*ppWriter)->pSmaFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosFsyncFile((*ppWriter)->pLastFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - } - - if (taosCloseFile(&(*ppWriter)->pHeadFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosCloseFile(&(*ppWriter)->pDataFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosCloseFile(&(*ppWriter)->pSmaFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosCloseFile(&(*ppWriter)->pLastFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree((*ppWriter)->aBuf[iBuf]); - } - taosMemoryFree(*ppWriter); -_exit: - *ppWriter = NULL; - return code; - -_err: - tsdbError("vgId:%d, data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { - int32_t code = 0; - int64_t n; - char hdr[TSDB_FHDR_SIZE]; - - // head ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutHeadFile(hdr, &pWriter->fHead); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - - n = taosLSeekFile(pWriter->pHeadFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // data ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutDataFile(hdr, &pWriter->fData); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - - n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // sma ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutSmaFile(hdr, &pWriter->fSma); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - - n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // sst ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutSstFile(hdr, &pWriter->fSst[pWriter->wSet.nSstF - 1]); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - - n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - return code; - -_err: - tsdbError("vgId:%d, update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size = 0; - int64_t n; - - // check - if (taosArrayGetSize(aBlockIdx) == 0) { - pHeadFile->offset = pHeadFile->size; - goto _exit; - } - - // prepare - size = sizeof(uint32_t); - for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx)); - } - size += sizeof(TSCKSUM); - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - n = tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx)); - } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); - - // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // update - pHeadFile->offset = pHeadFile->size; - pHeadFile->size += size; - -_exit: - tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", TD_VID(pWriter->pTsdb->pVnode), - pHeadFile->offset, size, taosArrayGetSize(aBlockIdx)); - return code; - -_err: - tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size; - int64_t n; - - ASSERT(mBlock->nItem > 0); - - // alloc - size = sizeof(uint32_t) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - n += tPutMapData(pWriter->aBuf[0] + n, mBlock); - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); - - // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // update - pBlockIdx->offset = pHeadFile->size; - pBlockIdx->size = size; - pHeadFile->size += size; - - tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 - " size:%" PRId64 " nItem:%d", - TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, - pBlockIdx->offset, pBlockIdx->size, mBlock->nItem); - return code; - -_err: - tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { - int32_t code = 0; - SSstFile *pSstFile = &pWriter->fSst[pWriter->wSet.nSstF - 1]; - int64_t size; - int64_t n; - - // check - if (taosArrayGetSize(aSstBlk) == 0) { - pSstFile->offset = pSstFile->size; - goto _exit; - } - - // size - size = sizeof(uint32_t); // TSDB_FILE_DLMT - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) { - size += tPutSstBlk(NULL, taosArrayGet(aSstBlk, iBlockL)); - } - size += sizeof(TSCKSUM); - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // encode - n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) { - n += tPutSstBlk(pWriter->aBuf[0] + n, taosArrayGet(aSstBlk, iBlockL)); - } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); - - // write - n = taosWriteFile(pWriter->pLastFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // update - pSstFile->offset = pSstFile->size; - pSstFile->size += size; - -_exit: - tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), - pSstFile->offset, size); - return code; - -_err: - tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SDataBlk *pDataBlk) { - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; - - if (iRow == 0) { - if (tsdbKeyCmprFn(&pDataBlk->minKey, &key) > 0) { - pDataBlk->minKey = key; - } - } else { - if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) { - pDataBlk->hasDup = 1; - } - } - - if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&pDataBlk->maxKey, &key) < 0) { - pDataBlk->maxKey = key; - } - - pDataBlk->minVer = TMIN(pDataBlk->minVer, key.version); - pDataBlk->maxVer = TMAX(pDataBlk->maxVer, key.version); - } - pDataBlk->nRow += pBlockData->nRow; -} - -static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { - int32_t code = 0; - - pSmaInfo->offset = 0; - pSmaInfo->size = 0; - - // encode - for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue; - - SColumnDataAgg sma; - tsdbCalcColDataSMA(pColData, &sma); - - code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); - if (code) goto _err; - pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma); - } - - // write - if (pSmaInfo->size) { - int32_t size = pSmaInfo->size + sizeof(TSCKSUM); - - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pSmaInfo->offset = pWriter->fSma.size; - pWriter->fSma.size += size; - } - - return code; - -_err: - tsdbError("vgId:%d tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast) { - int32_t code = 0; - - ASSERT(pBlockData->nRow > 0); - - pBlkInfo->offset = toLast ? pWriter->fSst[pWriter->wSet.nSstF - 1].size : pWriter->fData.size; - pBlkInfo->szBlock = 0; - pBlkInfo->szKey = 0; - - int32_t aBufN[4] = {0}; - code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN); - if (code) goto _err; - - // write ================= - TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; - - pBlkInfo->szKey = aBufN[3] + aBufN[2]; - pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - - int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], aBufN[3]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pFD, pWriter->aBuf[2], aBufN[2]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (aBufN[1]) { - n = taosWriteFile(pFD, pWriter->aBuf[1], aBufN[1]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - } - - if (aBufN[0]) { - n = taosWriteFile(pFD, pWriter->aBuf[0], aBufN[0]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - } - - // update info - if (toLast) { - pWriter->fSst[pWriter->wSet.nSstF - 1].size += pBlkInfo->szBlock; - } else { - pWriter->fData.size += pBlkInfo->szBlock; - } - - // ================= SMA ==================== - if (pSmaInfo) { - code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo); - if (code) goto _err; - } - -_exit: - tsdbTrace("vgId:%d tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d", - TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset, - pBlkInfo->szBlock); - return code; - -_err: - tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { - int32_t code = 0; - int64_t n; - int64_t size; - TdFilePtr pOutFD = NULL; // TODO - TdFilePtr PInFD = NULL; // TODO - char fNameFrom[TSDB_FILENAME_LEN]; - char fNameTo[TSDB_FILENAME_LEN]; - - // head - tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); - tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); - - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pHeadF->size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // data - tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); - tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); - - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pDataF->size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // sst - tsdbSstFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSstF[0], fNameFrom); - tsdbSstFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSstF[0], fNameTo); - - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aSstF[0]->size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // sma - tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); - tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); - - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pSmaF->size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - return code; - -_err: - tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - // SDelFWriter ==================================================== int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; @@ -1558,7 +1534,6 @@ _err: tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } - // SDelFReader ==================================================== struct SDelFReader { STsdb *pTsdb; From 696df47f05651f54b4944718ce015ca13a2af2d3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 17:17:46 +0800 Subject: [PATCH 06/34] more --- source/dnode/vnode/src/inc/tsdb.h | 8 ++++++++ source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 7 ------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0e9ec7bc24..3ce526ed7f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -592,6 +592,14 @@ struct STsdbReadSnap { STsdbFS fs; }; +typedef struct { + TdFilePtr pFD; + int32_t szPage; + int32_t nBuf; + uint8_t *pBuf; + int64_t pgno; +} STsdbFD; + struct SDataFWriter { STsdb *pTsdb; SDFileSet wSet; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 4bbbf24c47..21c9bae868 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -15,13 +15,6 @@ #include "tsdb.h" // =============== PAGE-WISE FILE =============== -typedef struct { - TdFilePtr pFD; - int32_t szPage; - int32_t nBuf; - uint8_t *pBuf; - int64_t pgno; -} STsdbFD; static int32_t tsdbOpenFile(const char *path, int32_t opt, STsdbFD *pFD) { int32_t code = 0; From 173a8366e35e4a5337d46ba33381c4f0aeda003c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 17:20:50 +0800 Subject: [PATCH 07/34] fix: compile warning --- source/dnode/vnode/src/inc/tsdb.h | 11 ++++++----- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 3ce526ed7f..30d1d103c1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -65,6 +65,7 @@ typedef struct SBlockInfo SBlockInfo; typedef struct SSmaInfo SSmaInfo; typedef struct SBlockCol SBlockCol; typedef struct SVersionRange SVersionRange; +typedef struct SLDataIter SLDataIter; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 @@ -635,15 +636,15 @@ typedef struct { } SRowInfo; typedef struct SMergeTree { - int8_t backward; - SRBTree rbt; - SArray *pIterList; - struct SLDataIter *pIter; + int8_t backward; + SRBTree rbt; + SArray *pIterList; + SLDataIter *pIter; } SMergeTree; int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange); -void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter); +void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 0589199d24..6fc7bcd2e1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -16,7 +16,7 @@ #include "tsdb.h" // SLDataIter ================================================= -typedef struct SLDataIter { +struct SLDataIter { SRBTreeNode node; SSstBlk *pSstBlk; SDataFReader *pReader; @@ -30,7 +30,7 @@ typedef struct SLDataIter { uint64_t uid; STimeWindow timeWindow; SVersionRange verRange; -} SLDataIter; +}; int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iSst, int8_t backward, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange) { From 3e55e72ab549b011e1a944f81a675c19a6ba230f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 14:31:32 +0800 Subject: [PATCH 08/34] more code --- source/dnode/vnode/src/inc/tsdb.h | 17 +- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 18 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 466 +++++++----------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 34 -- 4 files changed, 201 insertions(+), 334 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 30d1d103c1..7fc00420b2 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -196,7 +196,6 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol uint8_t **ppBuf); int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, uint8_t **ppBuf); -int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); @@ -605,10 +604,10 @@ struct SDataFWriter { STsdb *pTsdb; SDFileSet wSet; - TdFilePtr pHeadFD; - TdFilePtr pDataFD; - TdFilePtr pSmaFD; - TdFilePtr pLastFD; + STsdbFD *pHeadFD; + STsdbFD *pDataFD; + STsdbFD *pSmaFD; + STsdbFD *pLastFD; SHeadFile fHead; SDataFile fData; @@ -621,10 +620,10 @@ struct SDataFWriter { struct SDataFReader { STsdb *pTsdb; SDFileSet *pSet; - TdFilePtr pHeadFD; - TdFilePtr pDataFD; - TdFilePtr pSmaFD; - TdFilePtr aLastFD[TSDB_MAX_SST_FILE]; + STsdbFD *pHeadFD; + STsdbFD *pDataFD; + STsdbFD *pSmaFD; + STsdbFD *aLastFD[TSDB_MAX_SST_FILE]; uint8_t *aBuf[3]; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index b01d1b80d4..96901dc0ea 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -33,11 +33,9 @@ struct SLDataIter { SVersionRange verRange; }; -static SBlockData* getCurrentBlock(SLDataIter* pIter) { - return &pIter->bData[pIter->loadIndex]; -} +static SBlockData *getCurrentBlock(SLDataIter *pIter) { return &pIter->bData[pIter->loadIndex]; } -static SBlockData* getNextBlock(SLDataIter* pIter) { +static SBlockData *getNextBlock(SLDataIter *pIter) { pIter->loadIndex ^= 1; return getCurrentBlock(pIter); } @@ -150,9 +148,9 @@ void tLDataIterNextBlock(SLDataIter *pIter) { static void findNextValidRow(SLDataIter *pIter) { int32_t step = pIter->backward ? -1 : 1; - bool hasVal = false; - int32_t i = pIter->iRow; - SBlockData* pBlockData = getCurrentBlock(pIter); + bool hasVal = false; + int32_t i = pIter->iRow; + SBlockData *pBlockData = getCurrentBlock(pIter); for (; i < pBlockData->nRow && i >= 0; i += step) { if (pBlockData->aUid != NULL) { @@ -220,8 +218,8 @@ bool tLDataIterNextRow(SLDataIter *pIter) { return false; } - int32_t iBlockL = pIter->iSstBlk; - SBlockData* pBlockData = getCurrentBlock(pIter); + int32_t iBlockL = pIter->iSstBlk; + SBlockData *pBlockData = getCurrentBlock(pIter); if (pBlockData->nRow == 0 && pIter->pSstBlk != NULL) { // current block not loaded yet pBlockData = getNextBlock(pIter); @@ -306,7 +304,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); int32_t code = TSDB_CODE_OUT_OF_MEMORY; - struct SLDataIter *pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; + struct SLDataIter *pIterList[TSDB_DEFAULT_SST_FILE] = {0}; for (int32_t i = 0; i < pFReader->pSet->nSstF; ++i) { // open all last file code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 21c9bae868..51461cc74f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -15,9 +15,17 @@ #include "tsdb.h" // =============== PAGE-WISE FILE =============== +static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t opt, STsdbFD **ppFD) { + int32_t code = 0; + STsdbFD *pFD; -static int32_t tsdbOpenFile(const char *path, int32_t opt, STsdbFD *pFD) { - int32_t code = 0; + *ppFD = NULL; + + pFD = (STsdbFD *)taosMemoryCalloc(1, sizeof(*pFD)); + if (pFD == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } pFD->pFD = taosOpenFile(path, opt); if (pFD->pFD == NULL) { @@ -25,7 +33,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t opt, STsdbFD *pFD) { goto _exit; } - pFD->szPage = 4096; + pFD->szPage = szPage; pFD->pgno = 0; pFD->nBuf = 0; pFD->pBuf = taosMemoryMalloc(pFD->szPage); @@ -33,17 +41,21 @@ static int32_t tsdbOpenFile(const char *path, int32_t opt, STsdbFD *pFD) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } + *ppFD = pFD; _exit: return code; } -static void tsdbCloseFile(STsdbFD *pFD) { +static void tsdbCloseFile(STsdbFD **ppFD) { + STsdbFD *pFD = *ppFD; taosMemoryFree(pFD->pBuf); taosCloseFile(&pFD->pFD); + taosMemoryFree(pFD); + *ppFD = NULL; } -static int32_t tsdbSyncFile(STsdbFD *pFD) { +static int32_t tsdbFsyncFile(STsdbFD *pFD) { int32_t code = 0; if (taosFsyncFile(pFD->pFD) < 0) { @@ -140,11 +152,18 @@ _exit: return code; } +static int32_t tsdbLSeekFile(STsdbFD *pFD, int64_t offset) { + int32_t code = 0; + ASSERT(0); + return code; +} + // SDataFWriter ==================================================== int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; int32_t flag; int64_t n; + int32_t szPage = 4096; SDataFWriter *pWriter = NULL; char fname[TSDB_FILENAME_LEN]; char hdr[TSDB_FHDR_SIZE] = {0}; @@ -156,14 +175,12 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS goto _err; } pWriter->pTsdb = pTsdb; - pWriter->wSet = (SDFileSet){ - .diskId = pSet->diskId, - .fid = pSet->fid, - .pHeadF = &pWriter->fHead, - .pDataF = &pWriter->fData, - .pSmaF = &pWriter->fSma, - .nSstF = pSet->nSstF // - }; + pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, + .fid = pSet->fid, + .pHeadF = &pWriter->fHead, + .pDataF = &pWriter->fData, + .pSmaF = &pWriter->fSma, + .nSstF = pSet->nSstF}; pWriter->fHead = *pSet->pHeadF; pWriter->fData = *pSet->pDataF; pWriter->fSma = *pSet->pSmaF; @@ -173,19 +190,13 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS } // head - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname); - pWriter->pHeadFD = taosOpenFile(fname, flag); - if (pWriter->pHeadFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbOpenFile(fname, szPage, flag, &pWriter->pHeadFD); + if (code) goto _err; - n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE, NULL); + if (code) goto _err; ASSERT(n == TSDB_FHDR_SIZE); @@ -193,78 +204,49 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS // data if (pWriter->fData.size == 0) { - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; } else { - flag = TD_FILE_WRITE; + flag = TD_FILE_READ | TD_FILE_WRITE; } tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname); - pWriter->pDataFD = taosOpenFile(fname, flag); - if (pWriter->pDataFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbOpenFile(fname, szPage, flag, &pWriter->pDataFD); + if (code) goto _err; if (pWriter->fData.size == 0) { - n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - + code = tsdbWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE, NULL); + if (code) goto _err; pWriter->fData.size += TSDB_FHDR_SIZE; } else { - n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_END); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - ASSERT(n == pWriter->fData.size); + // code = tsdbLSeekFile(pWriter->pDataFD, 0, SEEK_END); + // if (code) goto _err; } // sma if (pWriter->fSma.size == 0) { - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; } else { - flag = TD_FILE_WRITE; + flag = TD_FILE_READ | TD_FILE_WRITE; } tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname); - pWriter->pSmaFD = taosOpenFile(fname, flag); - if (pWriter->pSmaFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbOpenFile(fname, szPage, flag, &pWriter->pSmaFD); + if (code) goto _err; if (pWriter->fSma.size == 0) { - n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE, NULL); + if (code) goto _err; pWriter->fSma.size += TSDB_FHDR_SIZE; } else { - n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_END); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - ASSERT(n == pWriter->fSma.size); + code = tsdbLSeekFile(pWriter->pSmaFD, 0); + if (code) goto _err; } // sst ASSERT(pWriter->fSst[pSet->nSstF - 1].size == 0); - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; tsdbSstFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSst[pSet->nSstF - 1], fname); - pWriter->pLastFD = taosOpenFile(fname, flag); - if (pWriter->pLastFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbOpenFile(fname, szPage, flag, &pWriter->pLastFD); + if (code) goto _err; + code = tsdbWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE, NULL); + if (code) goto _err; pWriter->fSst[pWriter->wSet.nSstF - 1].size += TSDB_FHDR_SIZE; *ppWriter = pWriter; @@ -284,46 +266,31 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { pTsdb = (*ppWriter)->pTsdb; if (sync) { - if (taosFsyncFile((*ppWriter)->pHeadFD) < 0) { + if (tsdbFsyncFile((*ppWriter)->pHeadFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (taosFsyncFile((*ppWriter)->pDataFD) < 0) { + if (tsdbFsyncFile((*ppWriter)->pDataFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (taosFsyncFile((*ppWriter)->pSmaFD) < 0) { + if (tsdbFsyncFile((*ppWriter)->pSmaFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (taosFsyncFile((*ppWriter)->pLastFD) < 0) { + if (tsdbFsyncFile((*ppWriter)->pLastFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } } - if (taosCloseFile(&(*ppWriter)->pHeadFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosCloseFile(&(*ppWriter)->pDataFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosCloseFile(&(*ppWriter)->pSmaFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosCloseFile(&(*ppWriter)->pLastFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + tsdbCloseFile(&(*ppWriter)->pHeadFD); + tsdbCloseFile(&(*ppWriter)->pDataFD); + tsdbCloseFile(&(*ppWriter)->pSmaFD); + tsdbCloseFile(&(*ppWriter)->pLastFD); for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { tFree((*ppWriter)->aBuf[iBuf]); @@ -346,70 +313,42 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { // head ============== memset(hdr, 0, TSDB_FHDR_SIZE); tPutHeadFile(hdr, &pWriter->fHead); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - n = taosLSeekFile(pWriter->pHeadFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbLSeekFile(pWriter->pHeadFD, 0); + if (code) goto _err; - n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE, NULL); + if (code) goto _err; // data ============== memset(hdr, 0, TSDB_FHDR_SIZE); tPutDataFile(hdr, &pWriter->fData); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbLSeekFile(pWriter->pDataFD, 0); + if (code) goto _err; - n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE, NULL); + if (code) goto _err; // sma ============== memset(hdr, 0, TSDB_FHDR_SIZE); tPutSmaFile(hdr, &pWriter->fSma); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbLSeekFile(pWriter->pSmaFD, 0); + if (code) goto _err; - n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE, NULL); + if (code) goto _err; // sst ============== memset(hdr, 0, TSDB_FHDR_SIZE); tPutSstFile(hdr, &pWriter->fSst[pWriter->wSet.nSstF - 1]); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbLSeekFile(pWriter->pLastFD, 0); + if (code) goto _err; - n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE, NULL); + if (code) goto _err; return code; @@ -431,11 +370,9 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { } // prepare - size = sizeof(uint32_t); for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx)); } - size += sizeof(TSCKSUM); // alloc code = tRealloc(&pWriter->aBuf[0], size); @@ -443,20 +380,14 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { // build n = 0; - n = tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx)); } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); + ASSERT(n == size); // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size, NULL); + if (code) goto _err; // update pHeadFile->offset = pHeadFile->size; @@ -481,24 +412,16 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBloc ASSERT(mBlock->nItem > 0); // alloc - size = sizeof(uint32_t) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); + size = tPutMapData(NULL, mBlock); code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // build - n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - n += tPutMapData(pWriter->aBuf[0] + n, mBlock); - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); + n = tPutMapData(pWriter->aBuf[0] + n, mBlock); // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size, NULL); + if (code) goto _err; // update pBlockIdx->offset = pHeadFile->size; @@ -519,7 +442,7 @@ _err: int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { int32_t code = 0; SSstFile *pSstFile = &pWriter->fSst[pWriter->wSet.nSstF - 1]; - int64_t size; + int64_t size = 0; int64_t n; // check @@ -529,11 +452,9 @@ int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { } // size - size = sizeof(uint32_t); // TSDB_FILE_DLMT for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) { size += tPutSstBlk(NULL, taosArrayGet(aSstBlk, iBlockL)); } - size += sizeof(TSCKSUM); // alloc code = tRealloc(&pWriter->aBuf[0], size); @@ -541,20 +462,13 @@ int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { // encode n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) { n += tPutSstBlk(pWriter->aBuf[0] + n, taosArrayGet(aSstBlk, iBlockL)); } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pLastFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pLastFD, pWriter->aBuf[0], size, NULL); + if (code) goto _err; // update pSstFile->offset = pSstFile->size; @@ -592,21 +506,14 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, // write if (pSmaInfo->size) { - int32_t size = pSmaInfo->size + sizeof(TSCKSUM); - - code = tRealloc(&pWriter->aBuf[0], size); + code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size); if (code) goto _err; - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pSmaFD, pWriter->aBuf[0], pSmaInfo->size, NULL); + if (code) goto _err; pSmaInfo->offset = pWriter->fSma.size; - pWriter->fSma.size += size; + // pWriter->fSma.size += size; } return code; @@ -631,37 +538,25 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock if (code) goto _err; // write ================= - TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; + STsdbFD *pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; pBlkInfo->szKey = aBufN[3] + aBufN[2]; pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], aBufN[3]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pFD, pWriter->aBuf[3], aBufN[3], NULL); + if (code) goto _err; - n = taosWriteFile(pFD, pWriter->aBuf[2], aBufN[2]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pFD, pWriter->aBuf[2], aBufN[2], NULL); + if (code) goto _err; if (aBufN[1]) { - n = taosWriteFile(pFD, pWriter->aBuf[1], aBufN[1]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pFD, pWriter->aBuf[1], aBufN[1], NULL); + if (code) goto _err; } if (aBufN[0]) { - n = taosWriteFile(pFD, pWriter->aBuf[0], aBufN[0]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pFD, pWriter->aBuf[0], aBufN[0], NULL); + if (code) goto _err; } // update info @@ -804,6 +699,7 @@ _err: int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; SDataFReader *pReader; + int32_t szPage = 4096; char fname[TSDB_FILENAME_LEN]; // alloc @@ -818,36 +714,24 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS // open impl // head tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); - pReader->pHeadFD = taosOpenFile(fname, TD_FILE_READ); - if (pReader->pHeadFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD); + if (code) goto _err; // data tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); - pReader->pDataFD = taosOpenFile(fname, TD_FILE_READ); - if (pReader->pDataFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD); + if (code) goto _err; // sma tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); - pReader->pSmaFD = taosOpenFile(fname, TD_FILE_READ); - if (pReader->pSmaFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD); + if (code) goto _err; // sst for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname); - pReader->aLastFD[iSst] = taosOpenFile(fname, TD_FILE_READ); - if (pReader->aLastFD[iSst] == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aLastFD[iSst]); + if (code) goto _err; } *ppReader = pReader; @@ -864,29 +748,17 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { if (*ppReader == NULL) goto _exit; // head - if (taosCloseFile(&(*ppReader)->pHeadFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + tsdbCloseFile(&(*ppReader)->pHeadFD); // data - if (taosCloseFile(&(*ppReader)->pDataFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + tsdbCloseFile(&(*ppReader)->pDataFD); // sma - if (taosCloseFile(&(*ppReader)->pSmaFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + tsdbCloseFile(&(*ppReader)->pSmaFD); // sst for (int32_t iSst = 0; iSst < (*ppReader)->pSet->nSstF; iSst++) { - if (taosCloseFile(&(*ppReader)->aLastFD[iSst]) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + tsdbCloseFile(&(*ppReader)->aLastFD[iSst]); } for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) { @@ -919,14 +791,14 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; - // seek - if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // // seek + // if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { + // code = TAOS_SYSTEM_ERROR(errno); + // goto _err; + // } // read - n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size); + n = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -982,14 +854,14 @@ int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) { code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; - // seek - if (taosLSeekFile(pReader->aLastFD[iSst], offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // // seek + // if (taosLSeekFile(pReader->aLastFD[iSst], offset, SEEK_SET) < 0) { + // code = TAOS_SYSTEM_ERROR(errno); + // goto _err; + // } // read - n = taosReadFile(pReader->aLastFD[iSst], pReader->aBuf[0], size); + n = tsdbReadFile(pReader->aLastFD[iSst], offset, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1040,14 +912,14 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; - // seek - if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // // seek + // if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { + // code = TAOS_SYSTEM_ERROR(errno); + // goto _err; + // } // read - n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size); + n = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1097,18 +969,18 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aCol code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; - // seek - int64_t n = taosLSeekFile(pReader->pSmaFD, pSmaInfo->offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < pSmaInfo->offset) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + // // seek + // int64_t n = taosLSeekFile(pReader->pSmaFD, pSmaInfo->offset, SEEK_SET); + // if (n < 0) { + // code = TAOS_SYSTEM_ERROR(errno); + // goto _err; + // } else if (n < pSmaInfo->offset) { + // code = TSDB_CODE_FILE_CORRUPTED; + // goto _err; + // } // read - n = taosReadFile(pReader->pSmaFD, pReader->aBuf[0], size); + int64_t n = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1148,11 +1020,12 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo tBlockDataClear(pBlockData); - TdFilePtr pFD = fromLast ? pReader->aLastFD[0] : pReader->pDataFD; // (todo) + STsdbFD *pFD = fromLast ? pReader->aLastFD[0] : pReader->pDataFD; // (todo) + + // todo: realloc pReader->aBuf[0] // uid + version + tskey - code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->aBuf[0], pBlkInfo->szKey, 1); - if (code) goto _err; + tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey); // todo SDiskDataHdr hdr; uint8_t *p = pReader->aBuf[0] + tGetDiskDataHdr(pReader->aBuf[0], &hdr); @@ -1192,8 +1065,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (hdr.szBlkCol > 0) { int64_t offset = pBlkInfo->offset + pBlkInfo->szKey; - code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[0], hdr.szBlkCol + sizeof(TSCKSUM), 1); - if (code) goto _err; + tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol + sizeof(TSCKSUM)); } SBlockCol blockCol = {.cid = 0}; @@ -1233,8 +1105,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + pBlockCol->offset; int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); - code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[1], size, 0); - if (code) goto _err; + tsdbReadFile(pFD, offset, pReader->aBuf[1], size); code = tsdbDecmprColData(pReader->aBuf[1], pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->aBuf[2]); if (code) goto _err; @@ -1321,8 +1192,7 @@ int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk int32_t code = 0; // read - code = tsdbReadAndCheck(pReader->aLastFD[iSst], pSstBlk->bInfo.offset, &pReader->aBuf[0], pSstBlk->bInfo.szBlock, 0); - if (code) goto _exit; + tsdbReadFile(pReader->aLastFD[iSst], pSstBlk->bInfo.offset, pReader->aBuf[0], pSstBlk->bInfo.szBlock); // decmpr code = tDecmprBlockData(pReader->aBuf[0], pSstBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); @@ -1708,3 +1578,37 @@ _err: tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } + +static int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) { + int32_t code = 0; + + // alloc + code = tRealloc(ppOut, size); + if (code) goto _exit; + + // seek + int64_t n = taosLSeekFile(pFD, offset, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + + // read + n = taosReadFile(pFD, *ppOut, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } else if (n < size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + + // check + if (toCheck && !taosCheckChecksumWhole(*ppOut, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + +_exit: + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 6937a27fe4..9b3094bb2c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -2153,37 +2153,3 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in _exit: return code; } - -int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) { - int32_t code = 0; - - // alloc - code = tRealloc(ppOut, size); - if (code) goto _exit; - - // seek - int64_t n = taosLSeekFile(pFD, offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } - - // read - n = taosReadFile(pFD, *ppOut, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - - // check - if (toCheck && !taosCheckChecksumWhole(*ppOut, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - -_exit: - return code; -} From fe20dea410d3a6365d8a7186617acd99e26310f0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 16:12:29 +0800 Subject: [PATCH 09/34] more code --- source/dnode/vnode/src/inc/tsdb.h | 5 ++--- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 20 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 7fc00420b2..a404a3c9bb 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -623,9 +623,8 @@ struct SDataFReader { STsdbFD *pHeadFD; STsdbFD *pDataFD; STsdbFD *pSmaFD; - STsdbFD *aLastFD[TSDB_MAX_SST_FILE]; - - uint8_t *aBuf[3]; + STsdbFD *aSstFD[TSDB_MAX_SST_FILE]; + uint8_t *aBuf[3]; }; typedef struct { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 51461cc74f..80fef30a24 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -14,6 +14,9 @@ */ #include "tsdb.h" + +#define TSDB_DEFAULT_PAGE_SIZE 4096 + // =============== PAGE-WISE FILE =============== static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t opt, STsdbFD **ppFD) { int32_t code = 0; @@ -163,7 +166,7 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS int32_t code = 0; int32_t flag; int64_t n; - int32_t szPage = 4096; + int32_t szPage = TSDB_DEFAULT_PAGE_SIZE; SDataFWriter *pWriter = NULL; char fname[TSDB_FILENAME_LEN]; char hdr[TSDB_FHDR_SIZE] = {0}; @@ -699,7 +702,7 @@ _err: int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; SDataFReader *pReader; - int32_t szPage = 4096; + int32_t szPage = TSDB_DEFAULT_PAGE_SIZE; char fname[TSDB_FILENAME_LEN]; // alloc @@ -711,7 +714,6 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS pReader->pTsdb = pTsdb; pReader->pSet = pSet; - // open impl // head tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD); @@ -730,7 +732,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS // sst for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname); - code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aLastFD[iSst]); + code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSstFD[iSst]); if (code) goto _err; } @@ -758,7 +760,7 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { // sst for (int32_t iSst = 0; iSst < (*ppReader)->pSet->nSstF; iSst++) { - tsdbCloseFile(&(*ppReader)->aLastFD[iSst]); + tsdbCloseFile(&(*ppReader)->aSstFD[iSst]); } for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) { @@ -855,13 +857,13 @@ int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) { if (code) goto _err; // // seek - // if (taosLSeekFile(pReader->aLastFD[iSst], offset, SEEK_SET) < 0) { + // if (taosLSeekFile(pReader->aSstFD[iSst], offset, SEEK_SET) < 0) { // code = TAOS_SYSTEM_ERROR(errno); // goto _err; // } // read - n = tsdbReadFile(pReader->aLastFD[iSst], offset, pReader->aBuf[0], size); + n = tsdbReadFile(pReader->aSstFD[iSst], offset, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1020,7 +1022,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo tBlockDataClear(pBlockData); - STsdbFD *pFD = fromLast ? pReader->aLastFD[0] : pReader->pDataFD; // (todo) + STsdbFD *pFD = fromLast ? pReader->aSstFD[0] : pReader->pDataFD; // (todo) // todo: realloc pReader->aBuf[0] @@ -1192,7 +1194,7 @@ int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk int32_t code = 0; // read - tsdbReadFile(pReader->aLastFD[iSst], pSstBlk->bInfo.offset, pReader->aBuf[0], pSstBlk->bInfo.szBlock); + tsdbReadFile(pReader->aSstFD[iSst], pSstBlk->bInfo.offset, pReader->aBuf[0], pSstBlk->bInfo.szBlock); // decmpr code = tDecmprBlockData(pReader->aBuf[0], pSstBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); From 57437312db5d2af564871ba24923904496a3af19 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 4 Sep 2022 16:54:07 +0800 Subject: [PATCH 10/34] other: rsma code optimization --- source/dnode/vnode/src/inc/sma.h | 1 + source/dnode/vnode/src/sma/smaCommit.c | 1 - source/dnode/vnode/src/sma/smaEnv.c | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 25 +++++++++++++------------ 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index e303faf7de..c9d1c3d015 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -148,6 +148,7 @@ struct SRSmaInfoItem { }; struct SRSmaInfo { + SSma *pSma; STSchema *pTSchema; int64_t suid; int64_t lastRecv; // ms diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 6d3db47b18..77bf5e9783 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -364,7 +364,6 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { } SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SRSmaInfoItem *pItem = NULL; // step 1: merge qTaskInfo and iQTaskInfo // lock diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 8bf79ca30e..013481ab3d 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -220,7 +220,7 @@ static void tRSmaInfoHashFreeNode(void *data) { if ((pItem = RSMA_INFO_ITEM((SRSmaInfo *)pRSmaInfo, 1)) && pItem->level) { taosHashRemove(smaMgmt.refHash, &pItem, POINTER_BYTES); } - tdFreeRSmaInfo(NULL, pRSmaInfo, true); + tdFreeRSmaInfo(pRSmaInfo->pSma, pRSmaInfo, true); } } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 6732403846..fce244bfb8 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -121,27 +121,27 @@ static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t l */ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { if (pInfo) { - int32_t vid = pSma ? SMA_VID(pSma) : -1; for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (isDeepFree && pItem->tmrId) { - smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", vid, pItem->tmrId, pInfo->suid, i + 1); + smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pItem->tmrId, + pInfo->suid, i + 1); taosTmrStopA(&pItem->tmrId); } if (isDeepFree && pInfo->taskInfo[i]) { - tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], vid, i + 1); + tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1); } else { - smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", vid, + smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma), pInfo->suid, i + 1); } if (pInfo->iTaskInfo[i]) { - tdRSmaQTaskInfoFree(&pInfo->iTaskInfo[i], vid, i + 1); + tdRSmaQTaskInfoFree(&pInfo->iTaskInfo[i], SMA_VID(pSma), i + 1); } else { - smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty iTaskInfo", vid, - pInfo->suid, i + 1); + smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty iTaskInfo", + SMA_VID(pSma), pInfo->suid, i + 1); } } if (isDeepFree) { @@ -377,7 +377,10 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; goto _err; } + pRSmaInfo->pSma = pSma; pRSmaInfo->pTSchema = pTSchema; + pRSmaInfo->suid = suid; + T_REF_INIT_VAL(pRSmaInfo, 1); if (!(pRSmaInfo->queue = taosOpenQueue())) { goto _err; } @@ -391,8 +394,6 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con if (!(pRSmaInfo->iQall = taosAllocateQall())) { goto _err; } - pRSmaInfo->suid = suid; - T_REF_INIT_VAL(pRSmaInfo, 1); if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) { goto _err; @@ -1586,8 +1587,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { if (!(pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaRef->refId))) { smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%" PRIi64 " refId:%d)", - smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove - taosHashRemove(smaMgmt.refHash, ¶m, POINTER_BYTES); + smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove + taosHashRemove(smaMgmt.refHash, ¶m, POINTER_BYTES); return; } @@ -1636,7 +1637,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { switch (fetchTriggerStat) { case TASK_TRIGGER_STAT_ACTIVE: { smaDebug("vgId:%d, rsma fetch task planned for level:%" PRIi8 " suid:%" PRIi64 " since stat is active", - SMA_VID(pSma), pItem->level, pRSmaInfo->suid); + SMA_VID(pSma), pItem->level, pRSmaInfo->suid); // async process pItem->fetchLevel = pItem->level; #if 0 From 8ec79c9d5c94c3a26aebd7169c3305974352ba51 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 17:31:20 +0800 Subject: [PATCH 11/34] more code --- source/dnode/vnode/src/inc/tsdb.h | 6 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 213 ++++++------------ 2 files changed, 72 insertions(+), 147 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a404a3c9bb..1818c3ec9f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -593,11 +593,13 @@ struct STsdbReadSnap { }; typedef struct { - TdFilePtr pFD; + char *path; int32_t szPage; + int32_t flag; + TdFilePtr pFD; + int64_t pgno; int32_t nBuf; uint8_t *pBuf; - int64_t pgno; } STsdbFD; struct SDataFWriter { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 80fef30a24..a6141311cf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -18,30 +18,38 @@ #define TSDB_DEFAULT_PAGE_SIZE 4096 // =============== PAGE-WISE FILE =============== -static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t opt, STsdbFD **ppFD) { +#define PAGE_CONTENT_SIZE(SIZE) ((SIZE) - sizeof(TSCKSUM)) +#define PAGE_OFFSET(PGNO, SIZE) (((PGNO)-1) * (SIZE)) +#define OFFSET_PGNO(OFFSET, SIZE) ((OFFSET) / (SIZE) + 1) + +static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { int32_t code = 0; STsdbFD *pFD; *ppFD = NULL; - pFD = (STsdbFD *)taosMemoryCalloc(1, sizeof(*pFD)); + pFD = (STsdbFD *)taosMemoryCalloc(1, sizeof(*pFD) + strlen(path) + 1); if (pFD == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - pFD->pFD = taosOpenFile(path, opt); + pFD->path = (char *)&pFD[1]; + strcpy(pFD->path, path); + pFD->szPage = szPage; + pFD->flag = flag; + pFD->pFD = taosOpenFile(path, flag); if (pFD->pFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _exit; } - pFD->szPage = szPage; pFD->pgno = 0; pFD->nBuf = 0; - pFD->pBuf = taosMemoryMalloc(pFD->szPage); + pFD->pBuf = taosMemoryMalloc(szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pFD); goto _exit; } *ppFD = pFD; @@ -102,12 +110,15 @@ _exit: static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { int32_t code = 0; - int64_t n = taosLSeekFile(pFD->pFD, pgno * pFD->szPage, SEEK_SET); + // seek + int64_t offset = PAGE_OFFSET(pgno, pFD->szPage); + int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _exit; } + // read n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); @@ -117,6 +128,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { goto _exit; } + // check if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { code = TSDB_CODE_FILE_CORRUPTED; goto _exit; @@ -128,27 +140,33 @@ _exit: return code; } -static int64_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t count) { +static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t count) { int32_t code = 0; + int64_t n; + int64_t pgno = OFFSET_PGNO(offset, pFD->szPage); + int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage); - int64_t pgno = offset / pFD->szPage; - int64_t n = 0; + ASSERT(pgno); if (pFD->pgno == pgno) { int64_t bOff = offset % pFD->szPage; - int64_t nRead = TMIN(pFD->szPage - bOff - sizeof(TSCKSUM), count); - memcpy(pBuf + n, pFD->pBuf + bOff, nRead); + int64_t nRead = TMIN(szPgCont - bOff, count); + + ASSERT(bOff < szPgCont); + + memcpy(pBuf, pFD->pBuf + bOff, nRead); n = nRead; + pgno++; } while (n < count) { code = tsdbReadFilePage(pFD, pgno); if (code) goto _exit; - pgno++; - - int64_t nRead = TMIN(pFD->szPage - sizeof(TSCKSUM), count - n); + int64_t nRead = TMIN(szPgCont, count - n); memcpy(pBuf + n, pFD->pBuf, nRead); + n += nRead; + pgno++; } _exit: @@ -747,7 +765,7 @@ _err: int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { int32_t code = 0; - if (*ppReader == NULL) goto _exit; + if (*ppReader == NULL) return code; // head tsdbCloseFile(&(*ppReader)->pHeadFD); @@ -767,8 +785,6 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { tFree((*ppReader)->aBuf[iBuf]); } taosMemoryFree(*ppReader); - -_exit: *ppReader = NULL; return code; @@ -778,49 +794,27 @@ _err: } int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { - int32_t code = 0; - int64_t offset = pReader->pSet->pHeadF->offset; - int64_t size = pReader->pSet->pHeadF->size - offset; - int64_t n; - uint32_t delimiter; + int32_t code = 0; + int64_t offset = pReader->pSet->pHeadF->offset; + int64_t size = pReader->pSet->pHeadF->size - offset; // todo taosArrayClear(aBlockIdx); - if (size == 0) { - goto _exit; - } + if (size == 0) return code; // alloc code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; - // // seek - // if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { - // code = TAOS_SYSTEM_ERROR(errno); - // goto _err; - // } - // read - n = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size); + if (code) goto _err; // decode - n = 0; - n = tGetU32(pReader->aBuf[0] + n, &delimiter); + uint32_t delimiter; + int64_t n = tGetU32(pReader->aBuf[0], &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); - while (n < size - sizeof(TSCKSUM)) { + while (n < size) { SBlockIdx blockIdx; n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx); @@ -829,10 +823,8 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { goto _err; } } + ASSERT(n == size); - ASSERT(n + sizeof(TSCKSUM) == size); - -_exit: return code; _err: @@ -841,65 +833,41 @@ _err: } int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) { - int32_t code = 0; - int64_t offset = pReader->pSet->aSstF[iSst]->offset; - int64_t size = pReader->pSet->aSstF[iSst]->size - offset; - int64_t n; - uint32_t delimiter; + int32_t code = 0; + int64_t offset = pReader->pSet->aSstF[iSst]->offset; + int64_t size = pReader->pSet->aSstF[iSst]->size - offset; // todo taosArrayClear(aSstBlk); - if (size == 0) { - goto _exit; - } + if (size == 0) return code; // alloc code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; - // // seek - // if (taosLSeekFile(pReader->aSstFD[iSst], offset, SEEK_SET) < 0) { - // code = TAOS_SYSTEM_ERROR(errno); - // goto _err; - // } - // read - n = tsdbReadFile(pReader->aSstFD[iSst], offset, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + code = tsdbReadFile(pReader->aSstFD[iSst], offset, pReader->aBuf[0], size); + if (code) goto _err; // decode - n = 0; - n = tGetU32(pReader->aBuf[0] + n, &delimiter); + uint32_t delimiter; + int64_t n = tGetU32(pReader->aBuf[0], &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); - while (n < size - sizeof(TSCKSUM)) { - SSstBlk blockl; - n += tGetSstBlk(pReader->aBuf[0] + n, &blockl); + while (n < size) { + SSstBlk sstBlk; + n += tGetSstBlk(pReader->aBuf[0] + n, &sstBlk); - if (taosArrayPush(aSstBlk, &blockl) == NULL) { + if (taosArrayPush(aSstBlk, &sstBlk) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } } + ASSERT(n == size); - ASSERT(n + sizeof(TSCKSUM) == size); - -_exit: return code; _err: - tsdbError("vgId:%d read blockl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d read sst blk failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } @@ -907,49 +875,26 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl int32_t code = 0; int64_t offset = pBlockIdx->offset; int64_t size = pBlockIdx->size; - int64_t n; - int64_t tn; // alloc code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; - // // seek - // if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { - // code = TAOS_SYSTEM_ERROR(errno); - // goto _err; - // } - // read - n = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size); + if (code) goto _err; // decode - n = 0; - uint32_t delimiter; - n += tGetU32(pReader->aBuf[0] + n, &delimiter); + int64_t n = tGetU32(pReader->aBuf[0], &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); - tn = tGetMapData(pReader->aBuf[0] + n, mBlock); + int64_t tn = tGetMapData(pReader->aBuf[0] + n, mBlock); if (tn < 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - n += tn; - ASSERT(n + sizeof(TSCKSUM) == size); + ASSERT(n + tn == size); return code; @@ -967,48 +912,26 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aCol taosArrayClear(aColumnDataAgg); // alloc - int32_t size = pSmaInfo->size + sizeof(TSCKSUM); + int32_t size = pSmaInfo->size; code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; - // // seek - // int64_t n = taosLSeekFile(pReader->pSmaFD, pSmaInfo->offset, SEEK_SET); - // if (n < 0) { - // code = TAOS_SYSTEM_ERROR(errno); - // goto _err; - // } else if (n < pSmaInfo->offset) { - // code = TSDB_CODE_FILE_CORRUPTED; - // goto _err; - // } - // read - int64_t n = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], size); + if (code) goto _err; // decode - n = 0; + int32_t n = 0; while (n < pSmaInfo->size) { SColumnDataAgg sma; - n += tGetColumnDataAgg(pReader->aBuf[0] + n, &sma); + if (taosArrayPush(aColumnDataAgg, &sma) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } } - + ASSERT(n == pSmaInfo->size); return code; _err: From dfe82efcd7ae22622cad248e61a84b9fc06da625 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 18:18:01 +0800 Subject: [PATCH 12/34] more code --- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 64 +++++++++---------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index a6141311cf..cd2c6e7051 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -18,9 +18,12 @@ #define TSDB_DEFAULT_PAGE_SIZE 4096 // =============== PAGE-WISE FILE =============== -#define PAGE_CONTENT_SIZE(SIZE) ((SIZE) - sizeof(TSCKSUM)) -#define PAGE_OFFSET(PGNO, SIZE) (((PGNO)-1) * (SIZE)) -#define OFFSET_PGNO(OFFSET, SIZE) ((OFFSET) / (SIZE) + 1) +#define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM)) +#define LOGIC_TO_FILE_OFFSET(OFFSET, PAGE) \ + ((OFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (OFFSET) % PAGE_CONTENT_SIZE(PAGE)) +#define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE)*PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE)) +#define PAGE_OFFSET(PGNO, PAGE) (((PGNO)-1) * (PAGE)) +#define OFFSET_PGNO(OFFSET, PAGE) ((OFFSET) / (PAGE) + 1) static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { int32_t code = 0; @@ -140,16 +143,17 @@ _exit: return code; } -static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t count) { +static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { int32_t code = 0; int64_t n; - int64_t pgno = OFFSET_PGNO(offset, pFD->szPage); + int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); + int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage); ASSERT(pgno); if (pFD->pgno == pgno) { - int64_t bOff = offset % pFD->szPage; - int64_t nRead = TMIN(szPgCont - bOff, count); + int64_t bOff = fOffset % pFD->szPage; + int64_t nRead = TMIN(szPgCont - bOff, size); ASSERT(bOff < szPgCont); @@ -158,11 +162,11 @@ static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t pgno++; } - while (n < count) { + while (n < size) { code = tsdbReadFilePage(pFD, pgno); if (code) goto _exit; - int64_t nRead = TMIN(szPgCont, count - n); + int64_t nRead = TMIN(szPgCont, size - n); memcpy(pBuf + n, pFD->pBuf, nRead); n += nRead; @@ -794,9 +798,10 @@ _err: } int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { - int32_t code = 0; - int64_t offset = pReader->pSet->pHeadF->offset; - int64_t size = pReader->pSet->pHeadF->size - offset; // todo + int32_t code = 0; + SHeadFile *pHeadFile = pReader->pSet->pHeadF; + int64_t offset = pHeadFile->offset; + int64_t size = pHeadFile->size - offset; taosArrayClear(aBlockIdx); if (size == 0) return code; @@ -810,10 +815,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { if (code) goto _err; // decode - uint32_t delimiter; - int64_t n = tGetU32(pReader->aBuf[0], &delimiter); - ASSERT(delimiter == TSDB_FILE_DLMT); - + int64_t n = 0; while (n < size) { SBlockIdx blockIdx; n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx); @@ -833,9 +835,10 @@ _err: } int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) { - int32_t code = 0; - int64_t offset = pReader->pSet->aSstF[iSst]->offset; - int64_t size = pReader->pSet->aSstF[iSst]->size - offset; // todo + int32_t code = 0; + SSstFile *pSstFile = pReader->pSet->aSstF[iSst]; + int64_t offset = pSstFile->offset; + int64_t size = pSstFile->size - offset; taosArrayClear(aSstBlk); if (size == 0) return code; @@ -849,10 +852,7 @@ int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) { if (code) goto _err; // decode - uint32_t delimiter; - int64_t n = tGetU32(pReader->aBuf[0], &delimiter); - ASSERT(delimiter == TSDB_FILE_DLMT); - + int64_t n = 0; while (n < size) { SSstBlk sstBlk; n += tGetSstBlk(pReader->aBuf[0] + n, &sstBlk); @@ -885,16 +885,12 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl if (code) goto _err; // decode - uint32_t delimiter; - int64_t n = tGetU32(pReader->aBuf[0], &delimiter); - ASSERT(delimiter == TSDB_FILE_DLMT); - - int64_t tn = tGetMapData(pReader->aBuf[0] + n, mBlock); - if (tn < 0) { + int64_t n = tGetMapData(pReader->aBuf[0], mBlock); + if (n < 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - ASSERT(n + tn == size); + ASSERT(n == size); return code; @@ -912,12 +908,11 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aCol taosArrayClear(aColumnDataAgg); // alloc - int32_t size = pSmaInfo->size; - code = tRealloc(&pReader->aBuf[0], size); + code = tRealloc(&pReader->aBuf[0], pSmaInfo->size); if (code) goto _err; // read - code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], size); + code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], pSmaInfo->size); if (code) goto _err; // decode @@ -1117,7 +1112,8 @@ int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk int32_t code = 0; // read - tsdbReadFile(pReader->aSstFD[iSst], pSstBlk->bInfo.offset, pReader->aBuf[0], pSstBlk->bInfo.szBlock); + code = tsdbReadFile(pReader->aSstFD[iSst], pSstBlk->bInfo.offset, pReader->aBuf[0], pSstBlk->bInfo.szBlock); + if (code) goto _exit; // decmpr code = tDecmprBlockData(pReader->aBuf[0], pSstBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); From 3607b18368a1600578c956f9921145ec48236c6a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 18:20:12 +0800 Subject: [PATCH 13/34] more code --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 3 --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 2 -- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 13 ------------- 4 files changed, 1 insertion(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 1818c3ec9f..02a929fe5e 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -267,7 +267,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMa int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk); int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData); -int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData); +int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData); // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index f58226083b..83809cd128 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -92,9 +92,6 @@ typedef struct { SArray *aDelData; // SArray } SCommitter; -extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *aSstBlk, - SBlockData *pBlockData); // todo - static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter); static int32_t tsdbCommitData(SCommitter *pCommitter); static int32_t tsdbCommitDel(SCommitter *pCommitter); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 96901dc0ea..8606148bcd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -114,8 +114,6 @@ void tLDataIterClose(SLDataIter *pIter) { taosMemoryFree(pIter); } -extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData); - void tLDataIterNextBlock(SLDataIter *pIter) { int32_t step = pIter->backward ? -1 : 1; pIter->iSstBlk += step; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index cd2c6e7051..03edfb0040 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1095,19 +1095,6 @@ _err: return code; } -int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) { - int32_t code = 0; - - code = tsdbReadBlockDataImpl(pReader, &pSstBlk->bInfo, 1, pBlockData); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - return code; -} - int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) { int32_t code = 0; From 28b56baa98cdd890dc1d655d8ffb0146e4dcd822 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 18:23:19 +0800 Subject: [PATCH 14/34] more code --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 15 +++++++++++---- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 02a929fe5e..1818c3ec9f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -267,7 +267,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMa int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk); int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData); -int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData); +int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData); // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 83809cd128..6f7a78ee46 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -442,7 +442,7 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { pIter->iSstBlk = 0; SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, 0); - code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, iSst, pSstBlk, &pIter->bData); + code = tsdbReadSstBlock(pCommitter->dReader.pReader, iSst, pSstBlk, &pIter->bData); if (code) goto _err; pIter->iRow = 0; @@ -1056,7 +1056,7 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { if (pIter->iSstBlk < taosArrayGetSize(pIter->aSstBlk)) { SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk); - code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, pIter->iSst, pSstBlk, &pIter->bData); + code = tsdbReadSstBlock(pCommitter->dReader.pReader, pIter->iSst, pSstBlk, &pIter->bData); if (code) goto _exit; pIter->iRow = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 8606148bcd..171e32007a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -221,7 +221,7 @@ bool tLDataIterNextRow(SLDataIter *pIter) { if (pBlockData->nRow == 0 && pIter->pSstBlk != NULL) { // current block not loaded yet pBlockData = getNextBlock(pIter); - code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData); + code = tsdbReadSstBlock(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData); if (code != TSDB_CODE_SUCCESS) { goto _exit; } @@ -245,7 +245,7 @@ bool tLDataIterNextRow(SLDataIter *pIter) { if (iBlockL != pIter->iSstBlk) { pBlockData = getNextBlock(pIter); - code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData); + code = tsdbReadSstBlock(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData); if (code) { goto _exit; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 03edfb0040..1a9d540af4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1095,18 +1095,25 @@ _err: return code; } -int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) { +int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) { int32_t code = 0; + // alloc + code = tRealloc(&pReader->aBuf[0], pSstBlk->bInfo.szBlock); + if (code) goto _err; + // read code = tsdbReadFile(pReader->aSstFD[iSst], pSstBlk->bInfo.offset, pReader->aBuf[0], pSstBlk->bInfo.szBlock); - if (code) goto _exit; + if (code) goto _err; // decmpr code = tDecmprBlockData(pReader->aBuf[0], pSstBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); - if (code) goto _exit; + if (code) goto _err; -_exit: + return code; + +_err: + tsdbError("vgId:%d tsdb read sst block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } From 3b0008a6252282adc925ebaa23ca1e52c1a8a30d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 18:42:23 +0800 Subject: [PATCH 15/34] more code --- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 30 ++++++++------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 40 +++---------------- 2 files changed, 23 insertions(+), 47 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 1a9d540af4..1433d7c0be 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -934,18 +934,20 @@ _err: return code; } -static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, int8_t fromLast, - SBlockData *pBlockData) { +static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, SBlockData *pBlockData) { int32_t code = 0; tBlockDataClear(pBlockData); - STsdbFD *pFD = fromLast ? pReader->aSstFD[0] : pReader->pDataFD; // (todo) - - // todo: realloc pReader->aBuf[0] + STsdbFD *pFD = pReader->pDataFD; // uid + version + tskey - tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey); // todo + code = tRealloc(&pReader->aBuf[0], pBlkInfo->szKey); + if (code) goto _err; + + code = tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey); + if (code) goto _err; + SDiskDataHdr hdr; uint8_t *p = pReader->aBuf[0] + tGetDiskDataHdr(pReader->aBuf[0], &hdr); @@ -978,14 +980,15 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (code) goto _err; p += hdr.szKey; - ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey - sizeof(TSCKSUM)); + ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey); // read and decode columns if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit; if (hdr.szBlkCol > 0) { int64_t offset = pBlkInfo->offset + pBlkInfo->szKey; - tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol + sizeof(TSCKSUM)); + code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol); + if (code) goto _err; } SBlockCol blockCol = {.cid = 0}; @@ -1022,10 +1025,11 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo } } else { // decode from binary - int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + pBlockCol->offset; - int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); + int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + pBlockCol->offset; + int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue; - tsdbReadFile(pFD, offset, pReader->aBuf[1], size); + code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size); + if (code) goto _err; code = tsdbDecmprColData(pReader->aBuf[1], pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->aBuf[2]); if (code) goto _err; @@ -1044,7 +1048,7 @@ _err: int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) { int32_t code = 0; - code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], 0, pBlockData); + code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], pBlockData); if (code) goto _err; if (pDataBlk->nSubBlock > 1) { @@ -1062,7 +1066,7 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData tBlockDataInitEx(&bData2, pBlockData); for (int32_t iSubBlock = 1; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { - code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], 0, &bData1); + code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], &bData1); if (code) { tBlockDataDestroy(&bData1, 1); tBlockDataDestroy(&bData2, 1); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 9b3094bb2c..8509c0c759 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1548,7 +1548,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, if (code) goto _exit; blockCol.offset = aBufN[0]; - aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; } code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); @@ -1556,15 +1556,8 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, hdr.szBlkCol += tPutBlockCol(aBuf[1] + hdr.szBlkCol, &blockCol); } - aBufN[1] = 0; - if (hdr.szBlkCol > 0) { - aBufN[1] = hdr.szBlkCol + sizeof(TSCKSUM); - - code = tRealloc(&aBuf[1], aBufN[1]); - if (code) goto _exit; - - taosCalcChecksumAppend(0, aBuf[1], aBufN[1]); - } + // SBlockCol + aBufN[1] = hdr.szBlkCol; // uid + version + tskey aBufN[2] = 0; @@ -1585,16 +1578,11 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, if (code) goto _exit; aBufN[2] += hdr.szKey; - aBufN[2] += sizeof(TSCKSUM); - code = tRealloc(&aBuf[2], aBufN[2]); - if (code) goto _exit; - // hdr aBufN[3] = tPutDiskDataHdr(NULL, &hdr); code = tRealloc(&aBuf[3], aBufN[3]); if (code) goto _exit; tPutDiskDataHdr(aBuf[3], &hdr); - taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], aBufN[3]), aBuf[2], aBufN[2]); // aggragate if (ppOut) { @@ -1626,10 +1614,6 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin // SDiskDataHdr n += tGetDiskDataHdr(pIn + n, &hdr); - if (!taosCheckChecksumWhole(pIn, n + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } ASSERT(hdr.delimiter == TSDB_FILE_DLMT); pBlockData->suid = hdr.suid; @@ -1657,7 +1641,7 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin code = tsdbDecmprData(pIn + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * hdr.nRow, &aBuf[0]); if (code) goto _exit; - n = n + hdr.szKey + sizeof(TSCKSUM); + n += hdr.szKey; // loop to decode each column data if (hdr.szBlkCol == 0) goto _exit; @@ -1679,8 +1663,8 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin if (code) goto _exit; } } else { - code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &blockCol, hdr.cmprAlg, - hdr.nRow, pColData, &aBuf[0]); + code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, + &aBuf[0]); if (code) goto _exit; } } @@ -2062,12 +2046,6 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol } size += pBlockCol->szValue; - // checksum - size += sizeof(TSCKSUM); - code = tRealloc(ppOut, nOut + size); - if (code) goto _exit; - taosCalcChecksumAppend(0, *ppOut + nOut, size); - _exit: return code; } @@ -2076,12 +2054,6 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in uint8_t **ppBuf) { int32_t code = 0; - int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole(pIn, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - ASSERT(pColData->cid == pBlockCol->cid); ASSERT(pColData->type == pBlockCol->type); pColData->smaOn = pBlockCol->smaOn; From b85bde9a43bd77f0988dfb1d861fd846e106ebb7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 19:59:21 +0800 Subject: [PATCH 16/34] more code --- source/dnode/vnode/src/inc/tsdb.h | 4 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 208 +++++++++--------- 2 files changed, 112 insertions(+), 100 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 1818c3ec9f..da74dc9828 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -598,8 +598,8 @@ typedef struct { int32_t flag; TdFilePtr pFD; int64_t pgno; - int32_t nBuf; uint8_t *pBuf; + int64_t szFile; } STsdbFD; struct SDataFWriter { @@ -609,7 +609,7 @@ struct SDataFWriter { STsdbFD *pHeadFD; STsdbFD *pDataFD; STsdbFD *pSmaFD; - STsdbFD *pLastFD; + STsdbFD *pSstFD; SHeadFile fHead; SDataFile fData; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 1433d7c0be..0f50714d3e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -48,13 +48,18 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd } pFD->szPage = szPage; pFD->pgno = 0; - pFD->nBuf = 0; pFD->pBuf = taosMemoryMalloc(szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pFD); goto _exit; } + if (taosStatFile(path, &pFD->szFile, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + ASSERT(pFD->szFile % szPage == 0); + pFD->szFile = pFD->szFile / szPage; *ppFD = pFD; _exit: @@ -69,42 +74,29 @@ static void tsdbCloseFile(STsdbFD **ppFD) { *ppFD = NULL; } -static int32_t tsdbFsyncFile(STsdbFD *pFD) { +static int32_t tsdbWriteFilePage(STsdbFD *pFD) { int32_t code = 0; - if (taosFsyncFile(pFD->pFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } + if (pFD->pgno > 0) { + int64_t n = taosLSeekFile(pFD->pFD, PAGE_OFFSET(pFD->pgno, pFD->szPage), SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } -_exit: - return code; -} + taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); -static int32_t tsdbWriteFile(STsdbFD *pFD, uint8_t *pBuf, int32_t nBuf, int64_t *offset) { - int32_t code = 0; + n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } - int32_t n = 0; - while (n < nBuf) { - int32_t remain = pFD->szPage - pFD->nBuf - sizeof(TSCKSUM); - int32_t size = TMIN(remain, nBuf - n); - - memcpy(pFD->pBuf + pFD->nBuf, pBuf + n, size); - n += size; - pFD->nBuf += size; - - if (pFD->nBuf + sizeof(TSCKSUM) == pFD->szPage) { - taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); - - int64_t n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } - - pFD->nBuf = 0; + if (pFD->szFile < pFD->pgno) { + pFD->szFile = pFD->szFile; } } + pFD->pgno = 0; _exit: return code; @@ -113,6 +105,8 @@ _exit: static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { int32_t code = 0; + ASSERT(pgno <= pFD->szFile); + // seek int64_t offset = PAGE_OFFSET(pgno, pFD->szPage); int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); @@ -143,6 +137,38 @@ _exit: return code; } +static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { + int32_t code = 0; + int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); + int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); + int64_t bOffset = fOffset % pFD->szPage; + int64_t n = 0; + + do { + if (pFD->pgno != pgno) { + code = tsdbWriteFilePage(pFD); + if (code) goto _exit; + + if (pgno < pFD->szFile) { + code = tsdbReadFilePage(pFD, pgno); + if (code) goto _exit; + } else { + pFD->pgno = pgno; + } + } + + int64_t nRead = TMIN(PAGE_CONTENT_SIZE(pFD->szPage) - bOffset, size - n); + memcpy(pFD->pBuf + bOffset, pBuf + n, nRead); + + pgno++; + bOffset = 0; + n += nRead; + } while (n < size); + +_exit: + return code; +} + static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { int32_t code = 0; int64_t n; @@ -177,9 +203,18 @@ _exit: return code; } -static int32_t tsdbLSeekFile(STsdbFD *pFD, int64_t offset) { +static int32_t tsdbFsyncFile(STsdbFD *pFD) { int32_t code = 0; - ASSERT(0); + + code = tsdbWriteFilePage(pFD); + if (code) goto _exit; + + if (taosFsyncFile(pFD->pFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + +_exit: return code; } @@ -220,11 +255,8 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS code = tsdbOpenFile(fname, szPage, flag, &pWriter->pHeadFD); if (code) goto _err; - code = tsdbWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE, NULL); + code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE); if (code) goto _err; - - ASSERT(n == TSDB_FHDR_SIZE); - pWriter->fHead.size += TSDB_FHDR_SIZE; // data @@ -237,12 +269,9 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS code = tsdbOpenFile(fname, szPage, flag, &pWriter->pDataFD); if (code) goto _err; if (pWriter->fData.size == 0) { - code = tsdbWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE, NULL); + code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE); if (code) goto _err; pWriter->fData.size += TSDB_FHDR_SIZE; - } else { - // code = tsdbLSeekFile(pWriter->pDataFD, 0, SEEK_END); - // if (code) goto _err; } // sma @@ -255,22 +284,19 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS code = tsdbOpenFile(fname, szPage, flag, &pWriter->pSmaFD); if (code) goto _err; if (pWriter->fSma.size == 0) { - code = tsdbWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE, NULL); + code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE); if (code) goto _err; pWriter->fSma.size += TSDB_FHDR_SIZE; - } else { - code = tsdbLSeekFile(pWriter->pSmaFD, 0); - if (code) goto _err; } // sst ASSERT(pWriter->fSst[pSet->nSstF - 1].size == 0); flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; tsdbSstFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSst[pSet->nSstF - 1], fname); - code = tsdbOpenFile(fname, szPage, flag, &pWriter->pLastFD); + code = tsdbOpenFile(fname, szPage, flag, &pWriter->pSstFD); if (code) goto _err; - code = tsdbWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE, NULL); + code = tsdbWriteFile(pWriter->pSstFD, 0, hdr, TSDB_FHDR_SIZE); if (code) goto _err; pWriter->fSst[pWriter->wSet.nSstF - 1].size += TSDB_FHDR_SIZE; @@ -291,31 +317,23 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { pTsdb = (*ppWriter)->pTsdb; if (sync) { - if (tsdbFsyncFile((*ppWriter)->pHeadFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbFsyncFile((*ppWriter)->pHeadFD); + if (code) goto _err; - if (tsdbFsyncFile((*ppWriter)->pDataFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbFsyncFile((*ppWriter)->pDataFD); + if (code) goto _err; - if (tsdbFsyncFile((*ppWriter)->pSmaFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbFsyncFile((*ppWriter)->pSmaFD); + if (code) goto _err; - if (tsdbFsyncFile((*ppWriter)->pLastFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbFsyncFile((*ppWriter)->pSstFD); + if (code) goto _err; } tsdbCloseFile(&(*ppWriter)->pHeadFD); tsdbCloseFile(&(*ppWriter)->pDataFD); tsdbCloseFile(&(*ppWriter)->pSmaFD); - tsdbCloseFile(&(*ppWriter)->pLastFD); + tsdbCloseFile(&(*ppWriter)->pSstFD); for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { tFree((*ppWriter)->aBuf[iBuf]); @@ -338,41 +356,25 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { // head ============== memset(hdr, 0, TSDB_FHDR_SIZE); tPutHeadFile(hdr, &pWriter->fHead); - - code = tsdbLSeekFile(pWriter->pHeadFD, 0); - if (code) goto _err; - - code = tsdbWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE, NULL); + code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE); if (code) goto _err; // data ============== memset(hdr, 0, TSDB_FHDR_SIZE); tPutDataFile(hdr, &pWriter->fData); - - code = tsdbLSeekFile(pWriter->pDataFD, 0); - if (code) goto _err; - - code = tsdbWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE, NULL); + code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE); if (code) goto _err; // sma ============== memset(hdr, 0, TSDB_FHDR_SIZE); tPutSmaFile(hdr, &pWriter->fSma); - - code = tsdbLSeekFile(pWriter->pSmaFD, 0); - if (code) goto _err; - - code = tsdbWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE, NULL); + code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE); if (code) goto _err; // sst ============== memset(hdr, 0, TSDB_FHDR_SIZE); tPutSstFile(hdr, &pWriter->fSst[pWriter->wSet.nSstF - 1]); - - code = tsdbLSeekFile(pWriter->pLastFD, 0); - if (code) goto _err; - - code = tsdbWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE, NULL); + code = tsdbWriteFile(pWriter->pSstFD, 0, hdr, TSDB_FHDR_SIZE); if (code) goto _err; return code; @@ -385,7 +387,7 @@ _err: int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { int32_t code = 0; SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size = 0; + int64_t size; int64_t n; // check @@ -395,6 +397,7 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { } // prepare + size = 0; for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx)); } @@ -411,7 +414,7 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { ASSERT(n == size); // write - code = tsdbWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size, NULL); + code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size); if (code) goto _err; // update @@ -442,10 +445,10 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBloc if (code) goto _err; // build - n = tPutMapData(pWriter->aBuf[0] + n, mBlock); + n = tPutMapData(pWriter->aBuf[0], mBlock); // write - code = tsdbWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size, NULL); + code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size); if (code) goto _err; // update @@ -467,7 +470,7 @@ _err: int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { int32_t code = 0; SSstFile *pSstFile = &pWriter->fSst[pWriter->wSet.nSstF - 1]; - int64_t size = 0; + int64_t size; int64_t n; // check @@ -477,6 +480,7 @@ int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { } // size + size = 0; for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) { size += tPutSstBlk(NULL, taosArrayGet(aSstBlk, iBlockL)); } @@ -492,7 +496,7 @@ int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { } // write - code = tsdbWriteFile(pWriter->pLastFD, pWriter->aBuf[0], size, NULL); + code = tsdbWriteFile(pWriter->pSstFD, pSstFile->size, pWriter->aBuf[0], size); if (code) goto _err; // update @@ -500,7 +504,7 @@ int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { pSstFile->size += size; _exit: - tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), + tsdbTrace("vgId:%d tsdb write sst block, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), pSstFile->offset, size); return code; @@ -534,11 +538,11 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size); if (code) goto _err; - code = tsdbWriteFile(pWriter->pSmaFD, pWriter->aBuf[0], pSmaInfo->size, NULL); + code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size); if (code) goto _err; pSmaInfo->offset = pWriter->fSma.size; - // pWriter->fSma.size += size; + pWriter->fSma.size += pSmaInfo->size; } return code; @@ -554,7 +558,11 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock ASSERT(pBlockData->nRow > 0); - pBlkInfo->offset = toLast ? pWriter->fSst[pWriter->wSet.nSstF - 1].size : pWriter->fData.size; + if (toLast) { + pBlkInfo->offset = pWriter->fSst[pWriter->wSet.nSstF - 1].size; + } else { + pBlkInfo->offset = pWriter->fData.size; + } pBlkInfo->szBlock = 0; pBlkInfo->szKey = 0; @@ -563,24 +571,28 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock if (code) goto _err; // write ================= - STsdbFD *pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; + STsdbFD *pFD = toLast ? pWriter->pSstFD : pWriter->pDataFD; pBlkInfo->szKey = aBufN[3] + aBufN[2]; pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - code = tsdbWriteFile(pFD, pWriter->aBuf[3], aBufN[3], NULL); + int64_t offset = pBlkInfo->offset; + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[3], aBufN[3]); if (code) goto _err; + offset += aBufN[3]; - code = tsdbWriteFile(pFD, pWriter->aBuf[2], aBufN[2], NULL); + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[2], aBufN[2]); if (code) goto _err; + offset += aBufN[2]; if (aBufN[1]) { - code = tsdbWriteFile(pFD, pWriter->aBuf[1], aBufN[1], NULL); + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[1], aBufN[1]); if (code) goto _err; + offset += aBufN[1]; } if (aBufN[0]) { - code = tsdbWriteFile(pFD, pWriter->aBuf[0], aBufN[0], NULL); + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], aBufN[0]); if (code) goto _err; } From 59912fedf116a28330d601a6d576f185fe9be4a0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 4 Sep 2022 20:14:21 +0800 Subject: [PATCH 17/34] fix: stream input data block memory leak --- source/dnode/vnode/src/sma/smaEnv.c | 3 ++- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- source/libs/executor/src/executor.c | 10 ++++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 013481ab3d..2595a629ba 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -62,7 +62,7 @@ int32_t smaInit() { } int32_t type = (8 == POINTER_BYTES) ? TSDB_DATA_TYPE_UBIGINT : TSDB_DATA_TYPE_UINT; - smaMgmt.refHash = taosHashInit(1, taosGetDefaultHashFunction(type), true, HASH_ENTRY_LOCK); + smaMgmt.refHash = taosHashInit(64, taosGetDefaultHashFunction(type), true, HASH_ENTRY_LOCK); if (!smaMgmt.refHash) { taosCloseRef(smaMgmt.rsetId); atomic_store_8(&smaMgmt.inited, 0); @@ -107,6 +107,7 @@ void smaCleanUp() { if (old == 1) { taosCloseRef(smaMgmt.rsetId); taosHashCleanup(smaMgmt.refHash); + smaMgmt.refHash = NULL; taosTmrCleanUp(smaMgmt.tmrHandle); smaInfo("sma mgmt env is cleaned up, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle); atomic_store_8(&smaMgmt.inited, 0); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index c8f3862071..a4b13e6a6b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1303,7 +1303,7 @@ _err: int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { int32_t code = 0; SLastFile *pLastFile = &pWriter->fLast; - int64_t size; + int64_t size = 0; int64_t n; // check diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 037bf543b1..136b2de596 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -30,6 +30,8 @@ static void cleanupRefPool() { taosCloseRef(ref); } +static FORCE_INLINE void streamInputBlockDataDestory(void* pBlock) { blockDataDestroy((SSDataBlock*)pBlock); } + static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) { ASSERT(pOperator != NULL); if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -53,7 +55,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu // prevent setting a different type of block pInfo->validBlockIndex = 0; if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { - taosArrayClearP(pInfo->pBlockLists, taosMemoryFree); + taosArrayClearP(pInfo->pBlockLists, streamInputBlockDataDestory); } else { taosArrayClear(pInfo->pBlockLists); } @@ -107,11 +109,7 @@ void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo) { if (pOptrInfo->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pInfo = pOptrInfo->info; if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { - for (int32_t i = 0; i < taosArrayGetSize(pInfo->pBlockLists); ++i) { - SSDataBlock* p = *(SSDataBlock**)taosArrayGet(pInfo->pBlockLists, i); - taosArrayDestroy(p->pDataBlock); - taosMemoryFreeClear(p); - } + taosArrayClearP(pInfo->pBlockLists, streamInputBlockDataDestory); } else { ASSERT(0); } From 7e1306a97657a752f45ae564270811cae968d8f6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 20:28:12 +0800 Subject: [PATCH 18/34] more code --- source/dnode/vnode/src/inc/tsdb.h | 38 ++-- source/dnode/vnode/src/tsdb/tsdbFS.c | 23 ++- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 194 ++++-------------- 3 files changed, 69 insertions(+), 186 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index da74dc9828..eb405fd4a6 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -67,11 +67,12 @@ typedef struct SBlockCol SBlockCol; typedef struct SVersionRange SVersionRange; typedef struct SLDataIter SLDataIter; -#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) -#define TSDB_MAX_SUBBLOCKS 8 -#define TSDB_MAX_SST_FILE 16 -#define TSDB_DEFAULT_SST_FILE 8 -#define TSDB_FHDR_SIZE 512 +#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) +#define TSDB_MAX_SUBBLOCKS 8 +#define TSDB_MAX_SST_FILE 16 +#define TSDB_DEFAULT_SST_FILE 8 +#define TSDB_FHDR_SIZE 512 +#define TSDB_DEFAULT_PAGE_SIZE 4096 #define HAS_NONE ((int8_t)0x1) #define HAS_NULL ((int8_t)0x2) @@ -578,20 +579,6 @@ struct SRowMerger { SArray *pArray; // SArray }; -struct SDelFWriter { - STsdb *pTsdb; - SDelFile fDel; - TdFilePtr pWriteH; - - uint8_t *aBuf[1]; -}; - -struct STsdbReadSnap { - SMemTable *pMem; - SMemTable *pIMem; - STsdbFS fs; -}; - typedef struct { char *path; int32_t szPage; @@ -602,6 +589,19 @@ typedef struct { int64_t szFile; } STsdbFD; +struct SDelFWriter { + STsdb *pTsdb; + SDelFile fDel; + STsdbFD *pWriteH; + uint8_t *aBuf[1]; +}; + +struct STsdbReadSnap { + SMemTable *pMem; + SMemTable *pIMem; + STsdbFS fs; +}; + struct SDataFWriter { STsdb *pTsdb; SDFileSet wSet; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index e6bc9d9936..0577faf855 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -15,12 +15,17 @@ #include "tsdb.h" +#define LOGIC_TO_FILE_SIZE(LSIZE, PAGE) (0) // todo + // ================================================================================================= static int32_t tsdbEncodeFS(uint8_t *p, STsdbFS *pFS) { int32_t n = 0; int8_t hasDel = pFS->pDelFile ? 1 : 0; uint32_t nSet = taosArrayGetSize(pFS->aDFileSet); + // version + n += tPutI8(p ? p + n : p, 0); + // SDelFile n += tPutI8(p ? p + n : p, hasDel); if (hasDel) { @@ -292,7 +297,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size != pSet->pHeadF->size) { + if (size != LOGIC_TO_FILE_SIZE(pSet->pHeadF->size, TSDB_DEFAULT_PAGE_SIZE)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -303,10 +308,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size < pSet->pDataF->size) { + if (size < LOGIC_TO_FILE_SIZE(pSet->pDataF->size, TSDB_DEFAULT_PAGE_SIZE)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; - } else if (size > pSet->pDataF->size) { + } else if (size > LOGIC_TO_FILE_SIZE(pSet->pDataF->size, TSDB_DEFAULT_PAGE_SIZE)) { code = tsdbDFileRollback(pTsdb, pSet, TSDB_DATA_FILE); if (code) goto _err; } @@ -317,10 +322,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size < pSet->pSmaF->size) { + if (size < LOGIC_TO_FILE_SIZE(pSet->pSmaF->size, TSDB_DEFAULT_PAGE_SIZE)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; - } else if (size > pSet->pSmaF->size) { + } else if (size > LOGIC_TO_FILE_SIZE(pSet->pSmaF->size, TSDB_DEFAULT_PAGE_SIZE)) { code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE); if (code) goto _err; } @@ -332,7 +337,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size != pSet->aSstF[iSst]->size) { + if (size != LOGIC_TO_FILE_SIZE(pSet->aSstF[iSst]->size, TSDB_DEFAULT_PAGE_SIZE)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -364,10 +369,12 @@ static int32_t tsdbRecoverFS(STsdb *pTsdb, uint8_t *pData, int64_t nData) { int32_t code = 0; int8_t hasDel; uint32_t nSet; - int32_t n; + int32_t n = 0; + + // version + n += tGetI8(pData + n, NULL); // SDelFile - n = 0; n += tGetI8(pData + n, &hasDel); if (hasDel) { pTsdb->fs.pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile)); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 0f50714d3e..91de4b3468 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -15,8 +15,6 @@ #include "tsdb.h" -#define TSDB_DEFAULT_PAGE_SIZE 4096 - // =============== PAGE-WISE FILE =============== #define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM)) #define LOGIC_TO_FILE_OFFSET(OFFSET, PAGE) \ @@ -1137,7 +1135,7 @@ _err: int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; char fname[TSDB_FILENAME_LEN]; - char hdr[TSDB_FHDR_SIZE] = {0}; + uint8_t hdr[TSDB_FHDR_SIZE] = {0}; SDelFWriter *pDelFWriter; int64_t n; @@ -1151,18 +1149,13 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb pDelFWriter->fDel = *pFile; tsdbDelFileName(pTsdb, pFile, fname); - pDelFWriter->pWriteH = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE); - if (pDelFWriter->pWriteH == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = + tsdbOpenFile(fname, TSDB_DEFAULT_PAGE_SIZE, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, &pDelFWriter->pWriteH); + if (code) goto _err; // update header - n = taosWriteFile(pDelFWriter->pWriteH, &hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE); + if (code) goto _err; pDelFWriter->fDel.size = TSDB_FHDR_SIZE; pDelFWriter->fDel.offset = 0; @@ -1182,16 +1175,13 @@ int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { STsdb *pTsdb = pWriter->pTsdb; // sync - if (sync && taosFsyncFile(pWriter->pWriteH) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (sync) { + code = tsdbFsyncFile(pWriter->pWriteH); + if (code) goto _err; } // close - if (taosCloseFile(&pWriter->pWriteH) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + tsdbCloseFile(&pWriter->pWriteH); for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) { tFree(pWriter->aBuf[iBuf]); @@ -1212,11 +1202,10 @@ int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelId int64_t n; // prepare - size = sizeof(uint32_t); + size = 0; for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData)); } - size += sizeof(TSCKSUM); // alloc code = tRealloc(&pWriter->aBuf[0], size); @@ -1224,22 +1213,14 @@ int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelId // build n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData)); } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); + ASSERT(n == size); // write - n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - ASSERT(n == size); + code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size); + if (code) goto _err; // update pDelIdx->offset = pWriter->fDel.size; @@ -1260,11 +1241,10 @@ int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { SDelIdx *pDelIdx; // prepare - size = sizeof(uint32_t); + size = 0; for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx)); } - size += sizeof(TSCKSUM); // alloc code = tRealloc(&pWriter->aBuf[0], size); @@ -1272,20 +1252,14 @@ int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { // build n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx)); } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); + ASSERT(n == size); // write - n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size); + if (code) goto _err; // update pWriter->fDel.offset = pWriter->fDel.size; @@ -1300,27 +1274,16 @@ _err: int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) { int32_t code = 0; - char hdr[TSDB_FHDR_SIZE]; + char hdr[TSDB_FHDR_SIZE] = {0}; int64_t size = TSDB_FHDR_SIZE; int64_t n; // build - memset(hdr, 0, size); tPutDelFile(hdr, &pWriter->fDel); - taosCalcChecksumAppend(0, hdr, size); - - // seek - if (taosLSeekFile(pWriter->pWriteH, 0, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } // write - n = taosWriteFile(pWriter->pWriteH, hdr, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pWriteH, 0, hdr, size); + if (code) goto _err; return code; @@ -1330,10 +1293,9 @@ _err: } // SDelFReader ==================================================== struct SDelFReader { - STsdb *pTsdb; - SDelFile fDel; - TdFilePtr pReadH; - + STsdb *pTsdb; + SDelFile fDel; + STsdbFD *pReadH; uint8_t *aBuf[1]; }; @@ -1355,14 +1317,9 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb pDelFReader->fDel = *pFile; tsdbDelFileName(pTsdb, pFile, fname); - pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ); - if (pDelFReader->pReadH == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - taosMemoryFree(pDelFReader); - goto _err; - } + code = tsdbOpenFile(fname, TSDB_DEFAULT_PAGE_SIZE, TD_FILE_READ, &pDelFReader->pReadH); + if (code) goto _err; -_exit: *ppReader = pDelFReader; return code; @@ -1377,10 +1334,7 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { SDelFReader *pReader = *ppReader; if (pReader) { - if (taosCloseFile(&pReader->pReadH) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } + tsdbCloseFile(&pReader->pReadH); for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(uint8_t *); iBuf++) { tFree(pReader->aBuf[iBuf]); } @@ -1400,38 +1354,17 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData taosArrayClear(aDelData); - // seek - if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - // alloc code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // read - n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size); + if (code) goto _err; // // decode n = 0; - - uint32_t delimiter; - n += tGetU32(pReader->aBuf[0] + n, &delimiter); - while (n < size - sizeof(TSCKSUM)) { + while (n < size) { SDelData delData; n += tGetDelData(pReader->aBuf[0] + n, &delData); @@ -1440,8 +1373,7 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData goto _err; } } - - ASSERT(n == size - sizeof(TSCKSUM)); + ASSERT(n == size); return code; @@ -1458,39 +1390,17 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { taosArrayClear(aDelIdx); - // seek - if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - // alloc code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // read - n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size); + if (code) goto _err; // decode n = 0; - uint32_t delimiter; - n += tGetU32(pReader->aBuf[0] + n, &delimiter); - ASSERT(delimiter == TSDB_FILE_DLMT); - - while (n < size - sizeof(TSCKSUM)) { + while (n < size) { SDelIdx delIdx; n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx); @@ -1501,45 +1411,11 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { } } - ASSERT(n == size - sizeof(TSCKSUM)); + ASSERT(n == size); return code; _err: tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; -} - -static int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) { - int32_t code = 0; - - // alloc - code = tRealloc(ppOut, size); - if (code) goto _exit; - - // seek - int64_t n = taosLSeekFile(pFD, offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } - - // read - n = taosReadFile(pFD, *ppOut, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - - // check - if (toCheck && !taosCheckChecksumWhole(*ppOut, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - -_exit: - return code; } \ No newline at end of file From bbd8eeaabe12e7e4bea790340058bfbfe31757b5 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 4 Sep 2022 20:47:22 +0800 Subject: [PATCH 19/34] fix: qtaskinfo file cleanup logic --- source/dnode/vnode/src/sma/smaCommit.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 77bf5e9783..fd300ef34a 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -188,13 +188,15 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) { for (int32_t i = 0; i < taosArrayGetSize(pFS->aQTaskInf);) { SQTaskFile *pTaskF = taosArrayGet(pFS->aQTaskInf, i); - if (atomic_sub_fetch_32(&pTaskF->nRef, 1) <= 0) { + int32_t oldVal = atomic_fetch_sub_32(&pTaskF->nRef, 1); + if ((oldVal <= 1) && (pTaskF->version < committed)) { tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); if (taosRemoveFile(qTaskInfoFullName) < 0) { - smaWarn("vgId:%d, cleanup qinf, failed to remove %s since %s", TD_VID(pVnode), qTaskInfoFullName, - tstrerror(TAOS_SYSTEM_ERROR(errno))); + smaWarn("vgId:%d, cleanup qinf, committed %" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed, + qTaskInfoFullName, tstrerror(TAOS_SYSTEM_ERROR(errno))); } else { - smaDebug("vgId:%d, cleanup qinf, success to remove %s", TD_VID(pVnode), qTaskInfoFullName); + smaDebug("vgId:%d, cleanup qinf, committed %" PRIi64 ", success to remove %s", TD_VID(pVnode), committed, + qTaskInfoFullName); } taosArrayRemove(pFS->aQTaskInf, i); continue; From 325b2e53278ec35893dde4f1981d5bf69ac0f847 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 21:38:19 +0800 Subject: [PATCH 20/34] fix more --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 91de4b3468..12ef4fd776 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -155,12 +155,12 @@ static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_ } } - int64_t nRead = TMIN(PAGE_CONTENT_SIZE(pFD->szPage) - bOffset, size - n); - memcpy(pFD->pBuf + bOffset, pBuf + n, nRead); + int64_t nWrite = TMIN(PAGE_CONTENT_SIZE(pFD->szPage) - bOffset, size - n); + memcpy(pFD->pBuf + bOffset, pBuf + n, nWrite); pgno++; bOffset = 0; - n += nRead; + n += nWrite; } while (n < size); _exit: @@ -169,12 +169,12 @@ _exit: static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { int32_t code = 0; - int64_t n; + int64_t n = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage); - ASSERT(pgno); + ASSERT(pgno && pgno <= pFD->szFile); if (pFD->pgno == pgno) { int64_t bOff = fOffset % pFD->szPage; int64_t nRead = TMIN(szPgCont - bOff, size); From 52e4a5e452671556808e0542c0c748af803c9d83 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 21:50:36 +0800 Subject: [PATCH 21/34] fix bug --- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 12ef4fd776..6b7b8efcb9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -173,28 +173,23 @@ static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage); + int64_t bOffset = fOffset % pFD->szPage; ASSERT(pgno && pgno <= pFD->szFile); - if (pFD->pgno == pgno) { - int64_t bOff = fOffset % pFD->szPage; - int64_t nRead = TMIN(szPgCont - bOff, size); - - ASSERT(bOff < szPgCont); - - memcpy(pBuf, pFD->pBuf + bOff, nRead); - n = nRead; - pgno++; - } + ASSERT(bOffset < szPgCont); while (n < size) { - code = tsdbReadFilePage(pFD, pgno); - if (code) goto _exit; + if (pFD->pgno != pgno) { + code = tsdbReadFilePage(pFD, pgno); + if (code) goto _exit; + } - int64_t nRead = TMIN(szPgCont, size - n); - memcpy(pBuf + n, pFD->pBuf, nRead); + int64_t nRead = TMIN(szPgCont - bOffset, size - n); + memcpy(pBuf + n, pFD->pBuf + bOffset, nRead); n += nRead; pgno++; + bOffset = 0; } _exit: From c71455216c09a817c7267b0182cb166ab671a344 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 22:29:59 +0800 Subject: [PATCH 22/34] more code --- source/dnode/vnode/src/inc/tsdb.h | 8 ++++++++ source/dnode/vnode/src/tsdb/tsdbFS.c | 2 -- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 7 ------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index eb405fd4a6..91aacaa328 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -84,6 +84,14 @@ typedef struct SLDataIter SLDataIter; #define TSDBKEY_MIN ((TSDBKEY){.ts = TSKEY_MIN, .version = VERSION_MIN}) #define TSDBKEY_MAX ((TSDBKEY){.ts = TSKEY_MAX, .version = VERSION_MAX}) +#define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM)) +#define LOGIC_TO_FILE_OFFSET(LOFFSET, PAGE) \ + ((LOFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (LOFFSET) % PAGE_CONTENT_SIZE(PAGE)) +#define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE)*PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE)) +#define PAGE_OFFSET(PGNO, PAGE) (((PGNO)-1) * (PAGE)) +#define OFFSET_PGNO(OFFSET, PAGE) ((OFFSET) / (PAGE) + 1) +#define LOGIC_TO_FILE_SIZE(LSIZE, PAGE) OFFSET_PGNO(LOGIC_TO_FILE_OFFSET(LSIZE, PAGE), PAGE) * (PAGE) + // tsdbUtil.c ============================================================================================== // TSDBROW #define TSDBROW_TS(ROW) (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow]) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 0577faf855..14bc1214a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -15,8 +15,6 @@ #include "tsdb.h" -#define LOGIC_TO_FILE_SIZE(LSIZE, PAGE) (0) // todo - // ================================================================================================= static int32_t tsdbEncodeFS(uint8_t *p, STsdbFS *pFS) { int32_t n = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 6b7b8efcb9..5911d59f20 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -16,13 +16,6 @@ #include "tsdb.h" // =============== PAGE-WISE FILE =============== -#define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM)) -#define LOGIC_TO_FILE_OFFSET(OFFSET, PAGE) \ - ((OFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (OFFSET) % PAGE_CONTENT_SIZE(PAGE)) -#define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE)*PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE)) -#define PAGE_OFFSET(PGNO, PAGE) (((PGNO)-1) * (PAGE)) -#define OFFSET_PGNO(OFFSET, PAGE) ((OFFSET) / (PAGE) + 1) - static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { int32_t code = 0; STsdbFD *pFD; From 04801585007d300c693e0a641eab87e81a97ef89 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 22:32:09 +0800 Subject: [PATCH 23/34] more fix --- source/dnode/vnode/src/tsdb/tsdbFile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 632a2c827b..2a7966e423 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -148,7 +148,7 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) { } // ftruncate - if (taosFtruncateFile(pFD, size) < 0) { + if (taosFtruncateFile(pFD, LOGIC_TO_FILE_SIZE(size, TSDB_DEFAULT_PAGE_SIZE)) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } From 70b89becb9c4e9afa2885398be31586b81221d75 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Sep 2022 23:51:26 +0800 Subject: [PATCH 24/34] more fix --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 5911d59f20..d42511e88a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -39,7 +39,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd } pFD->szPage = szPage; pFD->pgno = 0; - pFD->pBuf = taosMemoryMalloc(szPage); + pFD->pBuf = taosMemoryCalloc(1, szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pFD); @@ -84,7 +84,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD) { } if (pFD->szFile < pFD->pgno) { - pFD->szFile = pFD->szFile; + pFD->szFile = pFD->pgno; } } pFD->pgno = 0; From 20a269ef20f24539fc595a31149c6228200f7e94 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 5 Sep 2022 01:13:07 +0800 Subject: [PATCH 25/34] fix: more code --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index d42511e88a..e29920f626 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -408,8 +408,9 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { pHeadFile->size += size; _exit: - tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", TD_VID(pWriter->pTsdb->pVnode), - pHeadFile->offset, size, taosArrayGetSize(aBlockIdx)); + // tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", + // TD_VID(pWriter->pTsdb->pVnode), + // pHeadFile->offset, size, taosArrayGetSize(aBlockIdx)); return code; _err: @@ -779,8 +780,10 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { tsdbCloseFile(&(*ppReader)->pSmaFD); // sst - for (int32_t iSst = 0; iSst < (*ppReader)->pSet->nSstF; iSst++) { - tsdbCloseFile(&(*ppReader)->aSstFD[iSst]); + for (int32_t iSst = 0; iSst < TSDB_MAX_SST_FILE; iSst++) { + if ((*ppReader)->aSstFD[iSst]) { + tsdbCloseFile(&(*ppReader)->aSstFD[iSst]); + } } for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) { From 407773b67b0127413e45cddb6712d741e66d9289 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 5 Sep 2022 09:44:51 +0800 Subject: [PATCH 26/34] fix: double free for stream input block data --- source/libs/executor/src/executor.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 327242c4f7..95415e1113 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -30,8 +30,6 @@ static void cleanupRefPool() { taosCloseRef(ref); } -static FORCE_INLINE void streamInputBlockDataDestory(void* pBlock) { blockDataDestroy((SSDataBlock*)pBlock); } - static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) { ASSERT(pOperator != NULL); if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -55,7 +53,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu // prevent setting a different type of block pInfo->validBlockIndex = 0; if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { - taosArrayClearP(pInfo->pBlockLists, streamInputBlockDataDestory); + taosArrayClearP(pInfo->pBlockLists, taosMemoryFree); } else { taosArrayClear(pInfo->pBlockLists); } @@ -99,6 +97,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } +static FORCE_INLINE void streamInputBlockDataDestory(void* pBlock) { blockDataDestroy((SSDataBlock*)pBlock); } + void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; if (!pTaskInfo || !pTaskInfo->pRoot || pTaskInfo->pRoot->numOfDownstream <= 0) { From 7bbb86f17744906df1ebdfc6b04c1d2b337b5543 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 5 Sep 2022 10:19:17 +0800 Subject: [PATCH 27/34] fix: fix test case issue --- utils/test/c/sml_test.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 1fd1def263..ca3d464da7 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -92,7 +92,7 @@ int smlProcess_telnet_Test() { int smlProcess_json1_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); @@ -112,7 +112,7 @@ int smlProcess_json1_Test() { " }," " {" " \"metric\": \"sys.cpu.nice\"," - " \"timestamp\": 1346846400," + " \"timestamp\": 1662344042," " \"value\": 9," " \"tags\": {" " \"host\": \"web02\"," @@ -141,7 +141,7 @@ int smlProcess_json2_Test() { "{" " \"metric\": \"meter_current0\"," " \"timestamp\": {" - " \"value\" : 1346846400," + " \"value\" : 1662344042," " \"type\" : \"s\"" " }," " \"value\": {" @@ -181,7 +181,7 @@ int smlProcess_json3_Test() { "{" " \"metric\": \"meter_current1\"," " \"timestamp\": {" - " \"value\" : 1346846400," + " \"value\" : 1662344042," " \"type\" : \"s\"" " }," " \"value\": {" @@ -249,7 +249,7 @@ int smlProcess_json4_Test() { "{" " \"metric\": \"meter_current2\"," " \"timestamp\": {" - " \"value\" : 1346846500000," + " \"value\" : 1662344042000," " \"type\" : \"ms\"" " }," " \"value\": \"ni\"," From 06282bcbe078f60c4f55d3f908b4c233ea6442f6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 5 Sep 2022 10:19:31 +0800 Subject: [PATCH 28/34] fix(query): remove invalid assert. --- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 8 +++----- source/libs/executor/src/scanoperator.c | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f0518a72ab..8878bdcc99 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -881,7 +881,7 @@ void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLi void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput); -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, char** pNextStart); +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart); void updateLoadRemoteInfo(SLoadRemoteDataInfo *pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, SOperatorInfo* pOperator); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 39be1f7620..ec766913a9 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1866,13 +1866,11 @@ void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t pOperator->resultInfo.totalRows += numOfRows; } -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, - char** pNextStart) { +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) { if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); *pNextStart = (char*)blockDecode(pRes, pData); } else { // extract data according to pColList - ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; int32_t numOfCols = htonl(*(int32_t*)pStart); @@ -1970,7 +1968,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn char* pStart = pRetrieveRsp->data; while (index++ < pRetrieveRsp->numOfBlocks) { SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); - code = extractDataBlockFromFetchRsp(pb, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); + code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); goto _error; @@ -2095,7 +2093,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; char* pStart = pRetrieveRsp->data; - int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); + int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d4c98adb7c..e23a591904 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2539,7 +2539,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } char* pStart = pRsp->data; - extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pOperator->exprSupp.numOfExprs, pInfo->scanCols, &pStart); + extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->scanCols, &pStart); updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); // todo log the filter info From df7b597db01a91d5b49c4a9f165893196b2a9a6e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 5 Sep 2022 10:20:06 +0800 Subject: [PATCH 29/34] fix: invalid write --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index e29920f626..dec7d267a8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -988,6 +988,10 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (hdr.szBlkCol > 0) { int64_t offset = pBlkInfo->offset + pBlkInfo->szKey; + + code = tRealloc(&pReader->aBuf[0], hdr.szBlkCol); + if (code) goto _err; + code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol); if (code) goto _err; } @@ -1029,6 +1033,9 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + pBlockCol->offset; int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue; + code = tRealloc(&pReader->aBuf[1], size); + if (code) goto _err; + code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size); if (code) goto _err; From d82a9201be20e6863e8e77b2a35a30daef871296 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 5 Sep 2022 10:26:13 +0800 Subject: [PATCH 30/34] other:merge 3.0 --- CMakeSettings.json | 25 ------------------------- TDenginelogo.png | Bin 19663 -> 0 bytes examples/c/CMakeLists.txt | 15 --------------- 3 files changed, 40 deletions(-) delete mode 100644 CMakeSettings.json delete mode 100644 TDenginelogo.png diff --git a/CMakeSettings.json b/CMakeSettings.json deleted file mode 100644 index d3f2c27bf6..0000000000 --- a/CMakeSettings.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "configurations": [ - { - "name": "WSL-GCC-Debug", - "generator": "Unix Makefiles", - "configurationType": "Debug", - "buildRoot": "${projectDir}\\build\\", - "installRoot": "${projectDir}\\build\\", - "cmakeExecutable": "/usr/bin/cmake", - "cmakeCommandArgs": "", - "buildCommandArgs": "", - "ctestCommandArgs": "", - "inheritEnvironments": [ "linux_x64" ], - "wslPath": "${defaultWSLPath}", - "addressSanitizerRuntimeFlags": "detect_leaks=0", - "variables": [ - { - "name": "CMAKE_INSTALL_PREFIX", - "value": "/mnt/d/TDengine/TDengine/build", - "type": "PATH" - } - ] - } - ] -} \ No newline at end of file diff --git a/TDenginelogo.png b/TDenginelogo.png deleted file mode 100644 index 19a92592d7e8871778f5f3a6edd6314260d62551..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19663 zcmZ^~b95%bw>2Ewwr$&*IGH$^*tTukwmF%2V%s*J*v1om^Sk%HSnpb2|It;od+&4V zoa$cPU0ofetoQ>F4i63l1O!n=T0#W`1QhFET>%E_-`>PUZuVaRVIis@3IfuY0RLeG z`L9i4BCVnT0^&^t0umSs0s{O81)hR{xUqnMoEw6G@MVI4U_0ji{V4zf0tRoTrsb-o zAkSywV8>)+>R@cfwZMB-^@YwyD6DM;-B>zQm zwGkxKQcxxlcW^c%;bh`qVj&ZPBOxIXa5gpPQ<0GRANjwQAep7Bt0NyXvxkQVlLtGK zgR=!QD=#lEGYcCt8yn+42BV9Yy{nNYqrD6He;fHW;i>)Wb4(Lwz%lEgc2fxov)9jlKp^AJdER4K}AsAQ#Da3h$ksi9b z!hc%mDAF+UFzK*jnSp%}5c)1KOA(cS(66LJDi;rm_3*sJ5hyl`p$We%Hv}nL{`Z5Bq6Rm zN&2)9=L-F6P}Q7)`ZlQ!kn#%By**WnRpd&iqH!Zu8%xLf65H3YSyGcFiI&v!TGZ;= zH<7(Mt%EFfow8r8TcLqxVuuP}g0VUc$PNqwX*65i-yhBOt77PZk#g;8upj?=w5Gv*)RR5~9v5=y9i*0aeY zF!`S7NZ+CE9qT{!SG@!Mgb77uYg#A(-Cv*8t59;}dHE9d8T*@dp$L(6>j|tX+p)*0 z*0(;HSk4G(dzI&g`Uje6CPTCJ-AX2V0N5=%roinhxt#_ZtZ zv{gZK8EPnsUvOba)evx&>GK3CO}F!wRa1O}#il$q_~$?ON>vU3M(SZ%Ro>|(g3;z| zMRf=vcZaW}No84e>0k<_Q5N=i!d1ww+yv)55?6$%Mn>^^{8vj-CT zCB1u$BG@D|@rgXxMGIGv-l&4(t`c&MsN&Q!Sn7qGo(GyXDsoFGlR{n$)eF^-C z!5g1b~U6d67(RB?5(}#aVB3MsiNefST$k_JG|x`)UazSW%P$ zQdARXO=+{1y*^&=SARQP?6csSted9cx9*66xsH*+JAXyeyg z7Fe~ebPk9#?}+BtwNU0OQ3lGGI`J4dW!I~=R`^g!a`R&&y^_cAfj5g)>Ebo10sYSB z3MMOs0$wm{l0GZMt2Xo6#o z>w{g0V)gqc zwVj=c#*O@gix|iXLER8g&F6S24%UXxG(8sbgK_>ku0)m*He0miKo$8(>gHoJ9q>C#^_| zJ$tWMot6Is-rta`^etPV(|CGB4~9@43qrMXSz_L5R2`c+3bKj1;`dGATP101x^=V1 z)v%n&%%#jd>p-v+ii&S}>z!Qh%Ql7pT(4;R-w5c6*=#IL17|2VZjwZDA*{_aKKsv| z9zUN;dxbc+3mp87_;K&btS?|$;y}UBVync^rQ7H51K@$UBX+lP4Dpp`Z_#CWBAmj-OA z-sb}|r|bL0W^|f@1JVnDl?ZPTRF15}?P!|^__W|ws67jgc+zHg6b02GeEFyZy|(Bt ztWjNc-&Z-r)}hHf)$^!H*Nr`5sDgKKK{p&7r3;_&C<@@0v=r4nJNc~5V=ZYO-N z{BqI? zq1l|_>U{<_r(Vl)Ws2tVo@jl6{0Of4_i|^L5XuIMhe|3w_D&0Fh)A5Y0X5ys@s#g* ziG_YRmy>hYNSwI_vXdMJeg8P%`XEse5?&Auqgbt}k}H{&bqEYdWHI!w$`IU@rd~Bwy5UTKJ(K~-BZc5JN z5KDy?tzH|4kDZL0=$=7q8lV}SQF2cO0UkHUXS_wInE-kLH5}L+m?fGUa5vheAW9Z3 zsi#oWs`{!8oq91>NYF`98W?r{C^K*0%_H!n0HU=l46mTt zkrHCD+HSfZZQ+RN)tV8%g?$IK^!1p=(wSxFBgSleGCQKXXR4$UaDzlC( zzG$1@4}*_!1>|yezA!nl5Y{qg>WCi!m~H4*UE(A!{W4?}kTJqGxGNw+touckrNU9$ zR@Tb0Mq)B&#zj$&(Im>u1LBETzb9I&LvcH@n}iYS6UR@Yy~)5$t*0ovuN5?c!OWKm z7Qh$z-=H_bbR?5LAb@SSPq-1IJX=t0aavrhJs*N{H||9 z;cj|!WrXC7^2ddo*g-)sb$|QTy@RH4-ZQPPM5~$*M2SS35jO1lhgdI(@g^ z;)r;l0pP66b(|4gOpP^t8n^qpmDcFFZyOE~#fp{Dr^k3eZx7s0xx0)9UuQK{YRwvu zoR~`g$d?9Sf`%Tt- zq(h^_?#x<{fbVrWF9uz3FfH`$ZRnLEe&I%tOZOR=2I09bPZB1)QFXovYRrNHSA0@ZbMY zu~*rjrpklIt>x=%_c5vYOBBzXy6Rg9Wd{4F5A#}BXT~74M9o{Z@8sj@ z@FO8}0lqkY0oDb<8;vHB4l>6;)#X8{;IZ?tGIBdfX7c$%?tk-{ou_()DTB!g5rhe4 zLuqIK%n!Gcb;M&?pz8NEXH2rN-Cq06p1;+_?S6R^bMkyS>&qFs_pwMdniX@>h-soY zRWR(HLcn>LZ-mtBuw80g?y{K=%Y6fj&Mytk_G?*tnrxa}&&3R!!HXwGa+ffivnG(O zmd6E++%ow-dt6|u=U&FRF?d%08Xk6#D%=(5JlozNYTWG&({6*hGiP{{hxc`S#fnEke@RyQe@ z)Co8LbY&wl_FZR^L&v?5e1&lIcLwZebvpfvtAiVfXy~-&1-wBqx6#l(cgFv(%WO8` zB=(-CTC&(XuGbALL@cFb0*3;<+vu@Y&GXyaA6Hp;ELz0}8AH9_wp_oCPliI2E(fZh z2gOV&72-B;HVNo6a%TkLC|aVmaIw+k2K8L!yfAr0?KM3vg687V+RvWdmYQfYW?Qa) ziC1rCr_8Fls|fTQk+4e;Pd>--IGlC8W_jcjo-pOiE(l+O1>xbIM~C?rt4*pzr@rQ* zeGlfYo<@xptmA*b6l`OKe5|M~54l)<8U&mJwIpdk?!sAQ`Q2|Wh5|Z2tJ+U`M(3wY z5vmpAQg}+!gA!5Z0Q`WQHit;b#quF$|J#FhQ_71!EXOFH{i;uF#(W!yB^4LA;7d;x zNijl~Hk=0$!rJH_TfLmEy?xQT=)&8E$Sq{j^VQaIGIwJu*knXx`se9EJPzYquWv7U zIFrY-rXSYhml9u}Inm%)b*lP~suw>Tc({dpKV|c)Wk7Z@Q1)u1!!fKCVez@IB6mBJ zq9FROCQOm1wurn*{Vy_h&})o-=Qf}kBHqDoh+0xriDkb9IDF`DHB~WFk(2UWfj@Ro z&lqq~`(LnYCaDKSjX0A5*S^T?#@E@HqfEvc(~Q?fpoEy0<7&50hPONTsiUN%M<%2S5vNr2@rRXHYUj>j4*3Ty`yRRB_m+gZ#XtBX=`l@<|9Xi=-G zwL*9?IjEA9Ls(zX10G&>H*t15E&LG1DixjP#D3L*=s055%`Z+)4f}$qj;V3NJSN4x zA0+j5(K8tca`)U8wG|7JP-x*G!>EAHEtNWH)MQGMEi;$z)sSLLq{BnXqEEw9nH|;D zpz%*`eXZ5M<~Z!eKv9AhvWgZW;`|;wUUa{5w$v z*Alz+%RD%lZ|sU1`hN{(j67lIW;I#GLIIRt}WXW>WxB3*@1z%5k zcdy|VM#d~*xXa>Mq-teI@{B}$wsAkXD*c+zBLEQzLiirSPre_|6-+Uh@R85}Bf_q} zk=SWv0JzcB*=7y_gC!+xm@-)tjv*rmy{HY7-@5F>r#%^9t8~i2f~cHJ=;@ zla9ohM>7(rqE$g3C;59a75?BafwXSHBdHmIR<%|vslU$}LLEP{NH zDX&iE=Y0#wfQ7Io14274o*tgo1isVTVp8`n0!u(-2p)e6>FAZO%{NO&VN8@(e zh3)+wtA^qfy)7Q=%@;&-Pit?@F8ww1&2nw;Sb@+k)?w%RjX zMf{;nio3w{pz8W9pP(FXk+!qgW#<8d1N=j%2Tz-nka5eH+s04{G{26Ub!#J zxsLe;=24sMlC~Zzw{n^q+>ucAu^f5#v2aw*kmVr<14yapr;o7wbb4KWd_*vRvvA$; zflx{>D^{ASzc< z7AIfN6J)C@63GnK1$-I?zz0ktn_RSp1Pvr|0ZH60Zp^k}gT-E1R8QGigFVr}%DG4y ze}YGP5Uas2n^v-Ot~l(Dc5t+f_Jd{&ykMdCf zl7{{}MK6+G=RP>0gt|Vjy8yCd z(uXJ~Pljv>%r|N~UwTmRGjZ)ZY)q;@WXULd&TAD16d1e$E+T9IDL4GsaS{4rTMHge z$Fc(3dFxzJ=*VKStdLGihI{&_kKdm18GOVv6~htVkfT}gz;Pz(0{#avJ-kttA| z`>MM+N7V9hbR4pAq&L6)F2`fk4*A;LxX+*rWj|~7sf00;lXJ&sdHwaa9Nk#-k@qD+ zOL^^)!A`I_F3Lg`;h@njFeRH_RGz34HQL|}NPfp}(4PO5)*69QeP(h2rqEnF4=}Z$W(ug)rWB$Jk~n|>8~OK_jJm&c7oK?BDSiz6V2Y#HWUhu8aHh-54(O1=y;!R4@{Y6(wdOs^)}O9{uyvqs$eGr6vwGfpHyZmZ<51*f2-l%#f~r%vS`R^ zvKVd%xJSdmp*GXl(|R4h8HO*v=N8-%1=%;)zIWVm`*sh7M^bNr^Hyn4h`V}6`rU9| z)&5o7HV4$QLOHoK^!+^M>_b=2(eB;`2mHZv%haoPL9y(8Olli3t5GbWHy%nuX<^bU znAY-I$<^xkc=Y`C(HJK7d-`D;I6NPte7V`iJ>BIy2=4VnU%FyU*tgSITDqqalf(df zF%ZZ`nb0vi!a7j}`aBr>2x7G7L`o|hY6h0>`&dO&xzy)}+o`@;faKDU1Cwdt-qN>A zbP9d-hn3U+c_QrJbo)h$3jEyTK@11_&1Koiv$M)}S37W_`!2$un;xmZ);b<2s-qxp zbnkZojZ9{{`(xXEDL>9^84g(rNh%Qd*T?5oaHW^8>SI4XiQ+>sv5L!mz`JY~~lq~WYhrhTM( zCTGseD5HnHh*?AxCS9#>v*^&2)zj;)u#cUt+e|Q+4XwBsCa$wXbj_33Dr-ty zXT0aC7yM6(pEp5va9$+jNior6YE_xvTy>6rZmK*z^E@Wm0hD2kBbbh>89@fJ_7b;z zS*4~$QA>nPY|Xu{iHiY~vfz(x_+Wc);t|MfKUr`&U~oRukP8Tf8D3j)3_hueyBLK0x8e54)o^uIXTZ6RokWP3{70bLN*-2eGykgitt}$)Cz?$xg zmXNwF)*1I`NW-%L{8hnP;%psWvGFaf+xk#j`R$}nwTJpepvS7{Tpb`7M zbS&{;;)biD3}kGt2z5u4+4yqkt4||i3r{2nkjRaQiQ`-nM=nw+#LfC0)UkF{pp5>E z?mA6(eT{lC5HnD3A*K7^29F2j{7!25)h_;h=92GrUz{xS9Qy~oQ`@}zWJlp`sj!Xo zg9C1q3gNZ)Ym2kd7qtedl#JnRg?B8Bf_boFT;>ZwI#eNGACNKn>sV_PrhWo%=QUx7 zNJPc)OiS?rB2ahqCy7J%>m@5cMNTpMP}mS$osLkBQOz=m|2)f(1GK>+`b9qNy&`O) zD|hh*L{NUE&X(_P`l$;g$dwv#ngtx}BC>uj*p&-!m{wt7!Ld5s8FlKX;8u05G?>k) zCtAw6g|HVTZg)BQbA(UT*`Vqk6aev0!g~E+}h;&H*iK zbC(Z+3xk+S-A&XN!uP`IT1X5;rG6=p#{?3p@AADtn-DvfRZst*-0V%HP)o^b)kptA znXxygvgoGE7Iqf7tZC{B295`SK%^C8<*Xa8`WLa0J`pOa!17r>%EO?MEPZ0Qivg`xFR{8$xvYTC46a}2m9&;TTB%X| z*JLQt4M2W6cOWB1mq$9hTS#X5gjAu76^#j^L@wUHv$PgYG7%9(pAyr zNai%OsipH5d%}*;4rltm<}=l`CoB7XnNuhI>K{ z_W+nRSW*_hnLh>K=rp5pt)8FRvnZ?k%!;x@?KJ9?%g-WKZ3i(1N*7g`h-g>>9lr{8 zeNb=1kCR8V0N^(hm$s|HS($UmV2hR|Ja{-N_jTBL{SIuZJdo6H#G4{4)ExHX#(&fB z;-9lz9h;#mmIU`IfSHb_dth}im{$+Qj5L;Pa7|#bo~*qWtXfb`!a;7qzKJmm$Y>3^ zEj;D8#0e}(QyXh{*~n2obf?8D#B7b0sZr66R%o)q@DwI6sH@2{@&_m@2ND|KGE}ud z=h}uPG#H{9RB-1rug9?UW=gZ47BlIibXwc9M1OP|?-9uC&;GroYYM{%+JZN@DLIVi zqIH3q?r>wiLPnSq?s0th(Gq0CNkg62`oBS13~gCKG9l@ZHTQg!{l-Ko?IM3TX2z|7 zvE>|{baX8^<>+&*cQ5P}?N2yIbvVJ0Y(dJKGoe(9zh-OEne?(KfF&%{Bl&GJ@{zKF z4iElDA-_m2wyejOX8Aa{-TAByA+(VvLVIez>&%qdmY%#Q*QH%S`Kh;1H}nhp+ODY2R9af3X#c zmGUu4jUwfZPL~FDMX-q;2^l3xbWXD&2iph&a#7tO3tVOViouYSum+7}$gno5ie6d8 zr=Pn-xQt)Cz-2|=5|iTjH^lg*>D2fR_~Y^?@%fGk1>+SJ;^~8Szdr^zXlIit+a1o) ze9@WV(AMHQ@z4#@K*@9?+L>+nC)cMmshHpourZp0LTM%2Eztz^$*9HtK98q~M%`zu zbY;xeL%CZXwv20+$LMpm6bnk0Xt&p?$`jXiu&E~OT+jZ4%Q`7D=P@FwR@-?F!&)o7 z>jWSz?d@!>b-K$sIaMs@=))DdJeabIMv=&fy8KSHagTEj^vm@yGC*50o8Il#1mN&X zaCG8rGVHciJfg({E{ze(y(=<}?J6jy+WO3Od7(of!@9`)JO%l3(C9W(>jFu@#{ccd z!uzB{0iH*6^{5u+9fgHnfSoyezE0%t#npvHRb%{gX7n`*La@>g)8{3Z2G)%jcd|R5 z*iiGu3A30n@cVY^h3IrqS`MfrwKck$Y9Y7>T3z{12rU_kR*N@N`eOGs`_0+U>MRvS683k&kJ`y&qrUXWhxpN`>6F@RIm0{aX`0io8nn4{ZqVurzju;f(Yec-JE%fH@KgQNwu; z;OgxE71Q>+EBedQQ3H;A$@_rPn{P3)RgJ$$ks?!W(fcw7LaS^f_@Py5Dq`R1+Wy)Q z$b(+Cc0mw3c=_q5tqtkXd}su)?T?aK!oLI^XpwfGm=ir|WJPDWV{sdjL1Vzr2~2s? zBA&|E2{(aOscPV_ae@@h&fgqe;P{TMbh=6PDbB8i%L~fZYZeA3-$$*pur!wJY^XE>W@-fP(%h086VA@nsdKKrTq~#b=@-Eb;%#N)68qw z<;$-R2P4%F@z-cF-?Y5>q`x%&SO4Gh%k{NJQyCI|W2RTTwGW*goxR#iDT$Cp9oxN{ zCe;q41)bVL)5vZ$(BgAjApVfW!s1A5p!OlD%x~_u37%iozZK5eA-bZ5tgQTT)Y5Yx zw5{DbefC~>;+Fz5Ii@)-*eJWGo_p%VAOUZ_~(YNocxsQs1pyt2*_EmK>2+XH?m zaS_UHT6k*OgoJe^YCtHVjc#btI@KgZ2Rn$|q67AFS}iFthessL07RB4vRH$1a`i4A z@BNgIs+kRq7LGg@)I$yNZH)&Uj=cu9Dg#;^1Q$no3HZ2m@_OsaOS`XJt&s&!C9D2q zkOoWLcRfgXdMF?PRwfOlDa4IVhOPxr&@Ro)OBkXP0^my#aKci~jb6JW` zMADqd>DkTw2T#>X+t}V8LxmPqYaGg=rc#|LUF2ZSxYG>O{vK)jRA<1^&!26qCM5nv z#prW5grJpt^(oFv6orctY?||GDutc_WR!wO_NE{kVvJcs63d}NTd!FS{PW>nm{s*r z5HkxOY9R{@WYYJZ3pkmA`E)y7Ud~p?Bg~iEu_ru^mXl5I&S&*u!bQU?rn~e@2WlM; z>jE$vBBX&DX`+nkpHqH(a5=d$aRbdlD1ptw-?bf9tJOf98s$;1GpK%nOME2U7~xcY zx?UiyZ&L1icy2$n5@rX`Z-g7@+7geEH&9JsILA3%%|H4jOV}V#{yGMX*E?SFBx5M; z1d}~YQm=wSM^VM82_#r%DYHfNESsCV)GXx2q=8}Le35LrBaN1#hTq`Rz}rprs@3n+ zcrm&@0EG^&Bna?%*RhqW!X8a`!mRXVI@9}+dP;$0MQ1zU5-QQdG<|8M0WjrMw=9`DUfvAts4*-$F1kgP%j%HAB_#I*sreutx}aZlSkMBrFLMP|r29HK z3|4EJ*i~^jgM?-rJq2P|Hl-~jVy>glF83D`2-$4+1KT3r2?;qy_pb#Jk2ei%gR4TQ)GgjWb zRhUhilzvubyY^rgBZUo(`S=zT0i^{o55Brr&SWdNwcqZ|9A8x~%{#pl7S`yw1tV;h zj>o5;RvActiwC`knf}lj&IsD@jpbAzb3%rATYw?TzL?gNDZ^-I>`kWJ$ANfWducf6 z(k|SI;RB^*>A<4SRnd7@+9u~1Ke^=P&R7N+ocuX{P2r|xO9IMZm z3C>NtdV-IfUi=`k*xdaTa|+h|w#t+?Vo?Su24DovmrOX>za;ID6nx(OEzJ>tco$xZ z1?Iun)=*{#k<83ohcajfjimfwCZjX13qmsH39rBx;t3N78-_Xs(wVF1)D2^ zQxRxcvkO#DRjjuV}LujQ}fv3*2n|AnhPOw%FVI1d>y@(UQJ8^u#&5eIJzN(S&ZE(nv2oGX@ zf<7xUzZ(;b3eW-(1I$!FiIXh=VPyT`cuw-!)KQOJYhsUEBijGZwYI6CWGWEWDt|tl z%Ms{)nGmq7d8V=KakH*y%{zo5&B{FSi}hDJQ2}wy?BN_ku_%5X?d7BSC{vKmh}DPb zwmzRO&|xaH7k;PtphH+d(M`6YW4|QSX#V}$x0-Jtc(cMq;ef}fzLAk(EDR1zUZ!CGiTo%PVtEY&#Q7Pc+*8@wVrL}XeXB=Lv_0e%|lL&-sFBx7sAbji)TUmF1rZJ?2^4WTrFOu$~sJ!(P%pCv&XQGw84&9Z*p6&+BomyXg_pO3Y> z^U1Av(8!&sywxy*Qjk&|Z_{SmDR2aC)px%Z{P~u8V8a5rXv_jO452p`M|m``ugSqG z*s+afBX;us+27+K{=C=~tiXqfo{nh>B6RTT({)lxCq)Wp+j{6bF6FnH7POK1HPS)x z*$t<}0^aWw+>-2T@g(lf{>38#CB_s~9ov(#%m`wHQRl;tWk2npHHGYsrj? z7n**Xw0YEht^{2j9z6S)0>j~~3{`>Ur6my0$p3!u4gB|Mz0<=%hS|Gfw9HIK5Ys!Om?NL&_)0EyvjFpgU$@^pXPJo*rM-YmhE2Pmy$_jrM4Y@KFzN8(JliPM4q+ywXKuz$D^phIz)_NRn z&qL3N%aL+yag8K$%Ve~4(qXrg%l*bEM`j^ihr4Ya&t}M6M%DUs=bWnOLinNcJUs_< zeDN8?@hqM~AHm(%Y25d*I3bDSs~8%vUwjNI$#O~}DlBdZ)dN-XXWr`od}k_;f91E^&eZ-r53SkNpo3<`|5&F{oI4n{a2OGdesi3lwZv>5?;udt?;WE2@rdxY^88zrl z&@}noHS3?LDp)2?{8zURpJP^!4;qMKnNJd-fiO=jz`^--YV#5v;o;L$SFD|h)HtX+ zuR^U1%p(fkMdh@Hlp z&!5sp&nCbBE+8i<9T=XFKJN#@Z=A-R4WWM?S2V;*b{R-lRE^5HL&|0g7mB1MHju+o zc%HH+x!wiUqz|v!??&2!zFKPGR>h<{jh0-9UK)NV5%UNe2ptO{&d!e2fgT>!^7`2JX5E@x_z=z&+Wbs%}lL6 z$KqA)r78glKxe2Oc=-KanA=R8!BP%k#~+n#w>pHsw&R530=}vyO|I4SnUnG2@L=Ra z&5PTU@&!i*O2l%vRBA&CM%%8V8vNd))*iQZcuv?KXxPXVe7pWFL$MSb4gPM&VhDk# zdb3Z*8!GeL8HB1q?23N;ew!eKfHaCEK@cdUg@hFuZxYg8Bo6lEw%dRR$RCNV_=Yen zI-&v5!JzOq7?4xeA-6xw~3Paw1&;A^-ZT+cM;@=eS*rD4oq%%xO;-%7UV`zSNlp} zRFqb}@7stC$+wa8VaJDCPZ_7rwKf_*nv0Mp_9Y7IGg?M`~t&MXx+C zbSQ3@5)Va5HThyXWV~k4&qm*TKH@@z%x~m2Bk!-SU;ec_xqcP?K3(y=*_=IC`X27W z3^rjYFP;_sFwR8oZ8~FKgQiW}4P#a6h{JR(P z92i5x1rBsac&3e%Chm8BqKVJc2$!-_i3ca7cQ4wsUJB51qMA&to zmRT!zO+I_;{HCDYNL^rD@!f9o`AEN6KmnmU{<<+j<1ejNBP z?WN(YiSX5eB+rMRvPu@0Qbs!}BhgAg*eA(M!GLI~=brytZ7ni7-|0Ft^_o%8%k?!K z=OG#A!G>^%8j_zCggct-4xOU-a=3vwI}A{AFcDaaTGa#9em0@0=eU2||81AY=Z9a^ z0Ob^E7h2Sm`iW~3QPnM!R28Hwvrqvrc%3u5d-r)ac|dYcco^yDd#>oI;yc;ridB|S z^@@I3t=;_+8zAEumEzz1{Hc(?y#8_Q%MvEu|Giw%5T8E|^QbCt_eaB5VGdmfqLfIL ziW|T!^pfC-4tOzn;F_5h?R0-?<|^%-{Tx@FQwuj7y4eG6!tM7L7|F}>x$btClOUewQ%IM^oT4xQ&sG|3%OWUntPxtMCo%zVRg?#-rn)$?# z!!79lqKZ9QyIAp)xT((Z5K?16<$|V!U8I>RG2$kbusWL`v-E8N8G!Y4%aFd&|9b0Y z(J__^Sn{$FA8Dn??O@`|@pB1Ee+L=s=A|_}3!{fol$3g)>-(d${$>(z+KCop_E|%~ zK`9`}ECoX!Y{yL<2mzw$o~-tv5A*q9wDI@|!LlmG1y+Z4!rS`2pjc zB}Tqv;zzM&}e-H-M4Nj zHcW&W3%sz5f|3wNi{OlgvgagKK?n<`y0@f+fe=>h7z6Wt{p6$CJu*J8GWUtH_B^go z?dcpDo9%E>9DK|~ugSP=?RS|yRJICdB-1ZRwfRQk#$CVJJGy^3c=^u2*ILj==iY4* z@_bR_E3)ytj9E;2fdJ^Lvgjb?g7wQ<-b<)9Kr0-F%gsb4+=l<{Xqxm1a|_J=m`b_C zL__0%sM=yb7B6quFEmf$CM+b%;I*vC?4~fAm89KDQzv`?&r7gSBzc5H9Ka-m^!|gm ziKJt9h6Ca*L_=6TU3fsMPjj*9fv;s#sZ_z&lR* zr}Cf#Cx-xJLiyq{_H<0Z$JeA+Hx#`@712U(6)r!UL_e!7@j8{qWbOZ*A6|v=C=N9t1g~uc)ZLtj0a5-N^KE?g4VzKIpX~x znK=4b9%I~m{01N9X&afA6Q$OxmYPRNVWY)EMOD~9UF}LEcSAwJ6mpkA5%Jxkpp)v_ z*VAI_BgDsqdjIni;jf|>HDp4Aa*mRYX(2DHdJmpa>2LIanvAKC=b5MnPdPG5*oxC> z^s9xolIJfnLyOgLVYoCTgPzw2%Lk0+Butpww8@jv2$IlTM%-8sjqI6%D<9)Z@xTq1 zZvzG3+LJ|sP8W+aq>(e78o~&hukZdq7^_`$1bzY0BFc9kDg|o$NL!O3AzZNOkGgM9 zVk&e*3#e)3LdR^bDpX>x(YG$x^gRj=7)A4DbA8mJFeIkDxSA91x zYV@s{ujoqrZ2cK7S(PjG#~YG(g!~qu?D65fN*deHeG2~XFFJhV?c)#v<7WY}V-U)r zEFq~6nrF~BCy=i&*Ms6VhS;>2%)r@SGq_#(&NLdBU$3o?cV~e~FSrl4T1`*b0ejUb zU_X?Eq(%GN^4!gTqRG+Gnb1|&f>S5aIq4Zbu_uA{{jkf*Z7_2_NOy$0OU-s!UYX}l zj2=N_GX9CF3gs84Lb^p>IAilO6HkKF0oC60dOt|R))==dPGf%)(&>HZ%y3zlvlq<6 zg3>>z?^38!ZX2Bp9bU5q-vTvjh286+>;yrk91rYnu-!42#f}I!tI}Ocg`T#`nQ#quYa;@q@qV(Px%#8&Jhr1#ZnrtkW(en0ndDtq_y$YVH~omG@lt0j3~ zr?v)$F9R0d623c~bw8x9H}OVhhIQ>{mhK3o$^lZpwbCK1LDiBfY6?!uG@OijsMIfJ z`^{&AU_MZpU60FSC@@)tx|ld+Zmn28=91sLa>g{q9MWD)4%svHl-Ebq^G@HhDIy#VjyUe zw2IuD3&1Ah--{n1blnHqX64GwZhI<=p|AGhN>^-XZ) z$_Me|T4fVIg*(B@Y6RU{GPvcCp0vVNG9dP9k#b{5lFvTT_p(un zQ+cx{?#UBUnpwLpqry4^FK8*gq0E-KB^a&F;twt*4JGD=rSNF0=<{RZBi&B7b z6hUQumh=w|t6Pa=A$f3}TkQFn%z@ULZXzzy*`3S)+@Yx>t6Ca#C)<;r26{6wG7HO*MnZJqE1#e}of6@KXgC~JH9N49I5)J1u7 z>p1myR)SE_KUj7YbBk;mx&Vm#=HZWf`6U=IqjIw0hE;YoMQhJ@N+VRXEZ%}fT=aH5 zB|atP1VkCel@_m-`}QRs@f; z@T*N-q0+6VD^$FniGD}`5=8QcM+X3>Jhz4J8sV5TAY@txW|=QJ)Y9}3Z7@Ce5SK9@ zi4HL8Poq;up28*!5*-XnMlBb>kM=Urm%N=HrK-SRwb4 z{weELkaC!na4mu*;d7*tqyJKfP^X)chx;fWWqYHHT9{GW8As;;yFf5BJ6m!<<#7UBN&Z@-9MmWkRlTD1YLLRX+c~NAJJ9>tb#!B(ai#W z2uo;+DEqB5F9kjfr`9C___SLl$vpul+4)>y;EXDcBuZ6VMs5Q>9wn)L2;&@^^_V|9 z#!o4X8@3VD{0Dp(k^$zkiI+x z)391kHFrF*rSuD;S7f!&D9)VE3*LC9nOMeK4Xzq%3Onje{Mc{dkz5SBQjd_b)ts2f zH=MFBWQCAik&~HSN>`xSPKv&Y(NY-8fZ6lm8jkUU`m&h<)B?W>hOAx|KSrUE6o^%w zntZnE3*F^>1%~a3t33aw8gI#tK*_%8#sJXJQy$>N0p|&$YMg2^xhK!O;GoM}i2sDC zLMxRCX^S$#YP2-EV4;qj<+BD+Y@H5+`!WEx>;l6}!cQU>P&qI^N*R8+fXxtm<_D0{lpFJf1D=a;*50$h+e#PHr{GLQ|#BGKfS>9paxq zHG+o3j!COeZ^Oo2vHyiWm|LMIgZGnXQk=bfQ8rv)SdqSPf!kBNdkKy*$TIt8M0OFu zR*>074kP=ZP!R$!C}WVW)Rq(~nUxRAr4_FBDE!ArmJ`ncHtWcS%SFL`S%pm@o#Xcz zEY4S!Q-ic%0VuF2!XlgZ5&6VU>4};=<+!JISU6Te!%xS-0*omiWk0y6e0RdeY}qcq zT)C5OM@yp%76(vuD=W#4iulq&@A*6{>E_Mv^2EQx{;v zH(u?J(%#X8%5cryE>&$8=v7avzWH?oO;rJIj_i7&9x@K!1nR@T%l+;wbL4&(H{IG1 zN~F9RSvKz;?SnQlbJ3!XAA_w z2;V$!Cn>+QA|YpuslnmIgi$pk-7xy!?|c%cH+8VXxd(=8v5liibn@FTI|B}IyFc1t zG7MGF7hi>HdeM-*pEt`0(s(b~GH*9?DN~I%FNy|P&{@Wsx3g@8T_%oFu2glb&&*R5 zeo=ld3T!Bu^^U-GKI6UxszTKl*JUhSo-g;03~HGChYOVckL=&@BX6c(X(Vmt&4SO7 zg^u!D%ub{U2)r&3NHM0<)!7@GH|%Y)!o&$(>L*B$pFEZ!ju3eZA>CmATXh^oDi^|8ja=&cntNPM_-7q z{QdXhy2}Z6yK*_<;Rd;>ZIpSfgJ96nKGKvkJV2j@PUZsxgYn&;u8Y6Bbv@fOIEag2 zw!tCH9e3K+55$FME{=Cx=$jIXDWWsE>kvLQAU*y@IU!iaA5Ko59^q&;vb4r}s+-tu zyZ@>9@%>xl%rmCNWkh(MzkGVEShjeCa~qu{y{I~QYw4uDbHlT7$9+3v@GwVwqN((- zveqVgSHD(`K)UIZ8}ZAZxG1%oRDyVKaii_~t*bbzlLwk_s*y|xmNyl;-dvsKgbIum zv=NpvJr~44#WvBZ9u2W&+Y9ma+x4E@7pI>-HD*tr%yK-UgO!fw_a2Prckg4rjH@m> zVp)Z%yE_eIhr_;lJcoxJx6{yEe&d(l7qh4G%9gaI6gUVK^BLz;EUdhq_;_O>Fd_Y=_HOM7TzSA!P*R#y*YR_}eA4ZUH^kJ?FxxB1HmoQ60J<)gY zsqjf;Z4YKpqcI65a6d-`^v17!_Jgr(4!)Lj{!-2rBo~fx=nk7zrxe7;&tNA6%lN^? zN!ODaD>|3$-s$v3ud)A_ zH#%yE*in7us%7yjAHE3TVs4HDKqK;uo%Gol+kt2@W@bWR5k4m-jk*c(#B^z*)sqGS z$_e+n@b%=DeR0c=*T;iTJr{lYNQnIG@tqVdE2F8nOzd~YvEq5n_dFY+ixA2PVEif9 zT)sN4x?pKsuzWUxhcgs8*=!<>PY6s1mhp*~lcQHv1gdUS^%BHH95~d9r?>2lr#U|H z;jOzkWP5-U&Gy9rCrS7Y;?@wyiEzQ+z81#8n22_euI%(kYuxm%OEo zbfkP5P1XI0uWPP!my{RJ&_)Y={}T*iW=P0fCbi0!Nt=PTH|f3~T}d6A*ZVAn^YI X90YRHzn!lF00000NkvXXu0mjfHCr0+ diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt index 9d06dbac6d..4a9007acec 100644 --- a/examples/c/CMakeLists.txt +++ b/examples/c/CMakeLists.txt @@ -13,15 +13,9 @@ IF (TD_LINUX) #TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua) add_executable(tmq "") - add_executable(tmq_taosx "") add_executable(stream_demo "") add_executable(demoapi "") - target_sources(tmq_taosx - PRIVATE - "tmq_taosx.c" - ) - target_sources(tmq PRIVATE "tmq.c" @@ -41,10 +35,6 @@ IF (TD_LINUX) taos_static ) - target_link_libraries(tmq_taosx - taos_static - ) - target_link_libraries(stream_demo taos_static ) @@ -57,10 +47,6 @@ IF (TD_LINUX) PUBLIC "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) - target_include_directories(tmq_taosx - PUBLIC "${TD_SOURCE_DIR}/include/os" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" - ) target_include_directories(stream_demo PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" @@ -73,7 +59,6 @@ IF (TD_LINUX) ) SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) - SET_TARGET_PROPERTIES(tmq_taosx PROPERTIES OUTPUT_NAME tmq_taosx) SET_TARGET_PROPERTIES(stream_demo PROPERTIES OUTPUT_NAME stream_demo) SET_TARGET_PROPERTIES(demoapi PROPERTIES OUTPUT_NAME demoapi) ENDIF () From e66583be2091c33ba884c6b3b07ce4d25c85dd26 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 5 Sep 2022 10:27:09 +0800 Subject: [PATCH 31/34] other:merge 3.0 --- examples/c/tmq_taosx.c | 489 ----------------------------------------- 1 file changed, 489 deletions(-) delete mode 100644 examples/c/tmq_taosx.c diff --git a/examples/c/tmq_taosx.c b/examples/c/tmq_taosx.c deleted file mode 100644 index 491eda1ddb..0000000000 --- a/examples/c/tmq_taosx.c +++ /dev/null @@ -1,489 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include "taos.h" - -static int running = 1; - -static TAOS* use_db(){ - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return NULL; - } - - TAOS_RES* pRes = taos_query(pConn, "use db_taosx"); - if (taos_errno(pRes) != 0) { - printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes)); - return NULL; - } - taos_free_result(pRes); - return pConn; -} - -static void msg_process(TAOS_RES* msg) { - /*memset(buf, 0, 1024);*/ - printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); - printf("db: %s\n", tmq_get_db_name(msg)); - printf("vg: %d\n", tmq_get_vgroup_id(msg)); - TAOS *pConn = use_db(); - if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { - char* result = tmq_get_json_meta(msg); - if (result) { - printf("meta result: %s\n", result); - } - tmq_free_json_meta(result); - } - - tmq_raw_data raw = {0}; - tmq_get_raw(msg, &raw); - int32_t ret = tmq_write_raw(pConn, raw); - printf("write raw data: %s\n", tmq_err2str(ret)); - -// else{ -// while(1){ -// int numOfRows = 0; -// void *pData = NULL; -// taos_fetch_raw_block(msg, &numOfRows, &pData); -// if(numOfRows == 0) break; -// printf("write data: tbname:%s, numOfRows:%d\n", tmq_get_table_name(msg), numOfRows); -// int ret = taos_write_raw_block(pConn, numOfRows, pData, tmq_get_table_name(msg)); -// printf("write raw data: %s\n", tmq_err2str(ret)); -// } -// } - - taos_close(pConn); -} - -int32_t init_env() { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return -1; - } - - TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_taosx"); - if (taos_errno(pRes) != 0) { - printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database if not exists db_taosx vgroups 4"); - if (taos_errno(pRes) != 0) { - printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop database if exists abc1"); - if (taos_errno(pRes) != 0) { - printf("error in drop db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database if not exists abc1 vgroups 3"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, - "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " - "nchar(8), t4 bool)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct0 values(1626006833600, 1, 2, 'a')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct4 using st1(t3) tags('ct4')"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct4, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, 'ddd') ct0 values(1626006833602, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table st1 modify column c3 binary(64)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) (1626006833609, 51, 62, 'c333', 940)"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct3 select * from ct1"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table st1 add tag t2 binary(64)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table ct3 set tag t1=5000"); - if (taos_errno(pRes) != 0) { - printf("failed to slter child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "delete from abc1 .ct3 where ts < 1626006833606"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table ct3 ct1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); - if (taos_errno(pRes) != 0) { - printf("failed to create normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 add column c3 bigint"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 modify column c2 nchar(8)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 rename column c3 cc3"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 comment 'hello'"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 drop column c1"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into n1 values(now, 'eeee', 8989898899999) (now+9s, 'c333', 940)"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table n1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table jt1 using jt tags('{\"k1\":1, \"k2\":\"hello\"}')"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table jt2 using jt tags('')"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, - "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " - "nchar(8), t4 bool)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - taos_close(pConn); - return 0; -} - -int32_t create_topic() { - printf("create topic\n"); - TAOS_RES* pRes; - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return -1; - } - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - taos_close(pConn); - return 0; -} - -void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { - printf("commit %d tmq %p param %p\n", code, tmq, param); -} - -tmq_t* build_consumer() { -#if 0 - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); -#endif - - tmq_conf_t* conf = tmq_conf_new(); - tmq_conf_set(conf, "group.id", "tg2"); - tmq_conf_set(conf, "client.id", "my app 1"); - tmq_conf_set(conf, "td.connect.user", "root"); - tmq_conf_set(conf, "td.connect.pass", "taosdata"); - tmq_conf_set(conf, "msg.with.table.name", "true"); - tmq_conf_set(conf, "enable.auto.commit", "true"); - tmq_conf_set(conf, "experimental.snapshot.enable", "true"); - - - /*tmq_conf_set(conf, "experimental.snapshot.enable", "true");*/ - - tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); - tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); - assert(tmq); - tmq_conf_destroy(conf); - return tmq; -} - -tmq_list_t* build_topic_list() { - tmq_list_t* topic_list = tmq_list_new(); - tmq_list_append(topic_list, "topic_ctb_column"); - /*tmq_list_append(topic_list, "tmq_test_db_multi_insert_topic");*/ - return topic_list; -} - -void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - int32_t code; - - if ((code = tmq_subscribe(tmq, topics))) { - fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); - printf("subscribe err\n"); - return; - } - int32_t cnt = 0; - while (running) { - TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); - if (tmqmessage) { - cnt++; - msg_process(tmqmessage); - /*if (cnt >= 2) break;*/ - /*printf("get data\n");*/ - taos_free_result(tmqmessage); - /*} else {*/ - /*break;*/ - /*tmq_commit_sync(tmq, NULL);*/ - } - } - - code = tmq_consumer_close(tmq); - if (code) - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); - else - fprintf(stderr, "%% Consumer closed\n"); -} - -void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - static const int MIN_COMMIT_COUNT = 1; - - int msg_count = 0; - int32_t code; - - if ((code = tmq_subscribe(tmq, topics))) { - fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); - return; - } - - tmq_list_t* subList = NULL; - tmq_subscription(tmq, &subList); - char** subTopics = tmq_list_to_c_array(subList); - int32_t sz = tmq_list_get_size(subList); - printf("subscribed topics: "); - for (int32_t i = 0; i < sz; i++) { - printf("%s, ", subTopics[i]); - } - printf("\n"); - tmq_list_destroy(subList); - - while (running) { - TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); - if (tmqmessage) { - msg_process(tmqmessage); - taos_free_result(tmqmessage); - - /*tmq_commit_sync(tmq, NULL);*/ - /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ - } - } - - code = tmq_consumer_close(tmq); - if (code) - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); - else - fprintf(stderr, "%% Consumer closed\n"); -} - -int main(int argc, char* argv[]) { - printf("env init\n"); - if (init_env() < 0) { - return -1; - } - create_topic(); - - tmq_t* tmq = build_consumer(); - tmq_list_t* topic_list = build_topic_list(); - basic_consume_loop(tmq, topic_list); - /*sync_consume_loop(tmq, topic_list);*/ -} From d57129d78dff61fa5607a6d24d912a6af9232d0b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 5 Sep 2022 12:03:03 +0800 Subject: [PATCH 32/34] fix: another big bug --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index dec7d267a8..25daec76c6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -140,7 +140,7 @@ static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_ code = tsdbWriteFilePage(pFD); if (code) goto _exit; - if (pgno < pFD->szFile) { + if (pgno <= pFD->szFile) { code = tsdbReadFilePage(pFD, pgno); if (code) goto _exit; } else { From f35f5d5f4fea6b137de412a720c7d7aec75d6f9d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 5 Sep 2022 10:52:04 +0800 Subject: [PATCH 33/34] feat(stream): refactor single interval --- source/libs/executor/inc/executorimpl.h | 20 + source/libs/executor/src/executorimpl.c | 4 +- source/libs/executor/src/timewindowoperator.c | 488 ++++++++++++------ tests/script/tsim/stream/basic1.sim | 34 ++ .../tsim/stream/distributeInterval0.sim | 41 +- .../script/tsim/stream/partitionbyColumn0.sim | 28 +- .../script/tsim/stream/partitionbyColumn1.sim | 85 +-- .../script/tsim/stream/partitionbyColumn2.sim | 10 + 8 files changed, 492 insertions(+), 218 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 93bb5b0b49..a9826e0018 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -591,6 +591,24 @@ typedef struct SMergeAlignedIntervalAggOperatorInfo { SNode* pCondition; } SMergeAlignedIntervalAggOperatorInfo; +typedef struct SStreamIntervalOperatorInfo { + // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode + SOptrBasicInfo binfo; // basic info + SAggSupporter aggSup; // aggregate supporter + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; // multiple results build supporter + SInterval interval; // interval info + int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. + STimeWindowAggSupp twAggSup; + bool invertible; + bool ignoreExpiredData; + SArray* pRecycledPages; + SArray* pDelWins; // SWinRes + int32_t delIndex; + SSDataBlock* pDelRes; + bool isFinal; +} SStreamIntervalOperatorInfo; + typedef struct SStreamFinalIntervalOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; // basic info @@ -1003,6 +1021,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, + SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0240102437..205bcd58df 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3913,7 +3913,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, pScalarExprInfo, numOfScalarExpr, pAggNode->mergeDataBlock, pTaskInfo); } - } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { + } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); @@ -3938,6 +3938,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode, pTaskInfo, isStream); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { + pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode; pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b7a0673d88..d773f8a629 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -939,7 +939,7 @@ bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) { bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); } static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, - int32_t scanFlag, SHashObj* pUpdatedMap) { + int32_t scanFlag) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -955,21 +955,11 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder); - int32_t ret = TSDB_CODE_SUCCESS; - if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && - inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { - ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveWinResultRow(pResult, tableGroupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); - } + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, + numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - TSKEY ekey = ascScan ? win.ekey : win.skey; int32_t forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); @@ -991,12 +981,9 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup); } - if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && - inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, - numOfOutput); - } + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, + numOfOutput); doCloseWindow(pResultRowInfo, pInfo, pResult); @@ -1007,13 +994,6 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul if (startPos < 0) { break; } - if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) { - ekey = ascScan ? nextWin.ekey : nextWin.skey; - forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); - continue; - } - // null data, failed to allocate more memory buffer int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); @@ -1021,11 +1001,6 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveWinResultRow(pResult, tableGroupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); - } - ekey = ascScan ? nextWin.ekey : nextWin.skey; forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); @@ -1130,7 +1105,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, scanFlag, true); blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag); } initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->resultTsOrder); @@ -1581,141 +1556,6 @@ static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlo } } -static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { - SIntervalAggOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - pInfo->inputOrder = TSDB_ORDER_ASC; - SExprSupp* pSup = &pOperator->exprSupp; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - if (pOperator->status == OP_RES_TO_RETURN) { - doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); - if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "single interval"); - return pInfo->pDelRes; - } - - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - qDebug("===stream===single interval is done"); - freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); - } - printDataBlock(pInfo->binfo.pRes, "single interval"); - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; - } - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); - - SStreamState* pState = pTaskInfo->streamInfo.pState; - - while (1) { - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); - if (pBlock == NULL) { - break; - } - // qInfo("===stream===%ld", pBlock->info.version); - printDataBlock(pBlock, "single interval recv"); - - if (pBlock->info.type == STREAM_CLEAR) { - doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, - NULL); - qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); - continue; - } - if (pBlock->info.type == STREAM_DELETE_DATA) { - doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval, pUpdatedMap); - continue; - } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); - continue; - } - - if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) { - // set input version - pTaskInfo->version = pBlock->info.version; - } - - if (pInfo->scalarSupp.pExprInfo != NULL) { - SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); - } - - // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the - // caller. Note that all the time window are not close till now. - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, MAIN_SCAN, true); - if (pInfo->invertible) { - setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type); - } - - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); - } - -#if 0 - if (pState) { - printf(">>>>>>>> stream read backend\n"); - SWinKey key = { - .ts = 1, - .groupId = 2, - }; - char* val = NULL; - int32_t sz; - if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) { - ASSERT(0); - } - printf("stream read %s %d\n", val, sz); - streamFreeVal(val); - - SStreamStateCur* pCur = streamStateGetCur(pState, &key); - ASSERT(pCur); - while (streamStateCurNext(pState, pCur) == 0) { - SWinKey key1; - const void* val1; - if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) { - break; - } - printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz); - } - streamStateFreeCur(pCur); - } -#endif - - pOperator->status = OP_RES_TO_RETURN; - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, - pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); - - void* pIte = NULL; - while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { - taosArrayPush(pUpdated, pIte); - } - taosArraySort(pUpdated, resultrowComparAsc); - - finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - removeDeleteResults(pUpdatedMap, pInfo->pDelWins); - taosHashCleanup(pUpdatedMap); - doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); - if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "single interval"); - return pInfo->pDelRes; - } - - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - printDataBlock(pInfo->binfo.pRes, "single interval"); - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; -} - static void destroyStateWindowOperatorInfo(void* param) { SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); @@ -1925,7 +1765,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL, destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) { @@ -5627,3 +5467,311 @@ _error: pTaskInfo->code = code; return NULL; } + +static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, + SSDataBlock* pBlock, int32_t scanFlag, SHashObj* pUpdatedMap) { + SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; + + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + SExprSupp* pSup = &pOperatorInfo->exprSupp; + + int32_t startPos = 0; + int32_t numOfOutput = pSup->numOfExprs; + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); + TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; + uint64_t tableGroupId = pBlock->info.groupId; + bool ascScan = true; + TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); + SResultRow* pResult = NULL; + + STimeWindow win = + getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + int32_t ret = TSDB_CODE_SUCCESS; + if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && + inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { + ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, + numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); + setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); + } + } + + TSKEY ekey = ascScan ? win.ekey : win.skey; + int32_t forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + ASSERT(forwardRows > 0); + + if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && + inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, + numOfOutput); + } + + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = forwardRows - 1 + startPos; + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); + if (startPos < 0) { + break; + } + if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) { + ekey = ascScan ? nextWin.ekey : nextWin.skey; + forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + continue; + } + + // null data, failed to allocate more memory buffer + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); + setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); + } + + ekey = ascScan ? nextWin.ekey : nextWin.skey; + forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, + numOfOutput); + } +} + +static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int64_t maxTs = INT64_MIN; + SExprSupp* pSup = &pOperator->exprSupp; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); + if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, "single interval"); + return pInfo->pDelRes; + } + + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + qDebug("===stream===single interval is done"); + freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + } + printDataBlock(pInfo->binfo.pRes, "single interval"); + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; + } + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + + SStreamState* pState = pTaskInfo->streamInfo.pState; + + while (1) { + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + break; + } + printDataBlock(pBlock, "single interval recv"); + + if (pBlock->info.type == STREAM_CLEAR) { + doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, + NULL); + qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); + continue; + } else if (pBlock->info.type == STREAM_DELETE_DATA) { + doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval, pUpdatedMap); + continue; + } else if (pBlock->info.type == STREAM_GET_ALL) { + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); + continue; + } + + if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) { + // set input version + pTaskInfo->version = pBlock->info.version; + } + + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } + + // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the + // caller. Note that all the time window are not close till now. + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); + if (pInfo->invertible) { + setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type); + } + + maxTs = TMAX(maxTs, pBlock->info.window.ekey); + doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); + } + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); + +#if 0 + if (pState) { + printf(">>>>>>>> stream read backend\n"); + SWinKey key = { + .ts = 1, + .groupId = 2, + }; + char* val = NULL; + int32_t sz; + if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) { + ASSERT(0); + } + printf("stream read %s %d\n", val, sz); + streamFreeVal(val); + + SStreamStateCur* pCur = streamStateGetCur(pState, &key); + ASSERT(pCur); + while (streamStateCurNext(pState, pCur) == 0) { + SWinKey key1; + const void* val1; + if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) { + break; + } + printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz); + } + streamStateFreeCur(pCur); + } +#endif + + pOperator->status = OP_RES_TO_RETURN; + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, + pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + + void* pIte = NULL; + while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { + taosArrayPush(pUpdated, pIte); + } + taosArraySort(pUpdated, resultrowComparAsc); + + finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + removeDeleteResults(pUpdatedMap, pInfo->pDelWins); + taosHashCleanup(pUpdatedMap); + doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); + if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, "single interval"); + return pInfo->pDelRes; + } + + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + printDataBlock(pInfo->binfo.pRes, "single interval"); + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; +} + +void destroyStreamIntervalOperatorInfo(void* param) { + SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param; + cleanupBasicInfo(&pInfo->binfo); + cleanupAggSup(&pInfo->aggSup); + pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); + + pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + + cleanupGroupResInfo(&pInfo->groupResInfo); + colDataDestroy(&pInfo->twAggSup.timeWindowData); + taosMemoryFreeClear(param); +} + +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo) { + SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode; + + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); + ASSERT(numOfCols > 0); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + SInterval interval = {.interval = pIntervalPhyNode->interval, + .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, + .offset = pIntervalPhyNode->offset, + .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, }; + STimeWindowAggSupp twAggSupp = {.waterMark = pIntervalPhyNode->window.watermark, + .calTrigger = pIntervalPhyNode->window.triggerType, + .maxTs = INT64_MIN, }; + ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY); + pOperator->pTaskInfo = pTaskInfo; + pInfo->interval = interval; + pInfo->twAggSup = twAggSupp; + pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; + pInfo->isFinal = false; + + if (pIntervalPhyNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + + pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;; + initResultSizeInfo(&pOperator->resultInfo, 4096); + SExprSupp* pSup = &pOperator->exprSupp; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + initBasicInfo(&pInfo->binfo, pResBlock); + initStreamFunciton(pSup->pCtx, pSup->numOfExprs); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + pInfo->invertible = allInvertible(pSup->pCtx, numOfCols); + pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API + pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t)); + pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey)); + pInfo->delIndex = 0; + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); + initResultRowInfo(&pInfo->binfo.resultRowInfo); + + pOperator->name = "StreamIntervalOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, + destroyStreamIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + + initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark); + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + +_error: + destroyStreamIntervalOperatorInfo(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index d9777d5133..70ca1179b0 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -588,4 +588,38 @@ if $data00 != 5 then goto loop3 endi +#max,min selectivity +sql create database test3 vgroups 1; +sql use test3; +sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from ts1 interval(10s) ; + +sql insert into ts1 values(1648791211000,1,2,3); +sleep 50 +sql insert into ts1 values(1648791222001,2,2,3); +sleep 50 + +$loop_count = 0 +loop3: +sql select * from streamtST3; + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data02 != 1 then + print =====data02=$data02 + goto loop3 +endi + +# row 1 +if $data12 != 2 then + print =====data12=$data12 + goto loop3 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index b6b427343e..9b2e940556 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -198,7 +198,7 @@ endi sql select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5, avg(d) from st interval(10s); -sql create database test1 vgroups 1; +sql create database test1 vgroups 4; sql use test1; sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); @@ -232,4 +232,43 @@ if $data11 != 2 then goto loop2 endi +#max,min selectivity +sql create database test3 vgroups 4; +sql use test3; +sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ; + +sql insert into ts1 values(1648791211000,1,2,3); +sleep 50 +sql insert into ts1 values(1648791222001,2,2,3); +sleep 50 +sql insert into ts2 values(1648791211000,1,2,3); +sleep 50 +sql insert into ts2 values(1648791222001,2,2,3); +sleep 50 + +$loop_count = 0 +loop3: +sql select * from streamtST3; + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data02 != 1 then + print =====data02=$data02 + goto loop3 +endi + +# row 1 +if $data12 != 2 then + print =====data12=$data12 + goto loop3 +endi + system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/partitionbyColumn0.sim b/tests/script/tsim/stream/partitionbyColumn0.sim index d91d4b7bf0..24fdb9c994 100644 --- a/tests/script/tsim/stream/partitionbyColumn0.sim +++ b/tests/script/tsim/stream/partitionbyColumn0.sim @@ -24,7 +24,7 @@ sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); $loop_count = 0 loop0: -sleep 100 +sleep 50 sql select * from streamt order by c1, c4, c2, c3; $loop_count = $loop_count + 1 @@ -48,7 +48,7 @@ sql insert into t1 values(1648791213000,1,2,3,1.0); $loop_count = 0 loop1: -sleep 100 +sleep 50 sql select * from streamt order by c1, c4, c2, c3; $loop_count = $loop_count + 1 @@ -71,7 +71,7 @@ sql insert into t1 values(1648791213000,2,2,3,1.0); $loop_count = 0 loop2: -sleep 100 +sleep 50 sql select * from streamt order by c1, c4, c2, c3; $loop_count = $loop_count + 1 @@ -97,7 +97,7 @@ sql insert into t1 values(1648791213002,1,2,3,1.0); $loop_count = 0 loop3: -sleep 100 +sleep 50 sql select * from streamt order by c1, c4, c2, c3; $loop_count = $loop_count + 1 @@ -134,7 +134,7 @@ sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (16 $loop_count = 0 loop4: -sleep 100 +sleep 50 sql select * from streamt order by c1, c4, c2, c3; $loop_count = $loop_count + 1 @@ -208,7 +208,7 @@ sql insert into t1 values(1648791213001,1,2,3,2.0); $loop_count = 0 loop5: -sleep 100 +sleep 50 sql select * from streamt1 order by c1, c4, c2, c3; $loop_count = $loop_count + 1 @@ -229,7 +229,7 @@ sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0); $loop_count = 0 loop6: -sleep 100 +sleep 50 sql select * from streamt1 order by c1, c4, c2, c3; $loop_count = $loop_count + 1 @@ -294,7 +294,7 @@ sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); $loop_count = 0 loop7: -sleep 100 +sleep 50 sql select * from test.streamt2 order by c1, c2, c3; $loop_count = $loop_count + 1 @@ -318,7 +318,7 @@ sql insert into t2 values(1648791213000,1,2,3,1.0); $loop_count = 0 loop8: -sleep 100 +sleep 50 sql select * from test.streamt2 order by c1, c2, c3; $loop_count = $loop_count + 1 @@ -342,7 +342,7 @@ sql insert into t2 values(1648791213000,2,2,3,1.0); $loop_count = 0 loop9: -sleep 100 +sleep 50 sql select * from test.streamt2 order by c1, c2, c3; $loop_count = $loop_count + 1 @@ -372,7 +372,7 @@ sql insert into t2 values(1648791213002,1,2,3,1.0); $loop_count = 0 loop10: -sleep 100 +sleep 50 sql select * from test.streamt2 order by c1, c2, c3; $loop_count = $loop_count + 1 @@ -414,7 +414,7 @@ sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (16 $loop_count = 0 loop11: -sleep 100 +sleep 50 sql select * from test.streamt2 order by c1, c2, c3; $loop_count = $loop_count + 1 @@ -492,7 +492,7 @@ sql insert into t4 values(1648791213000,1,2,3,1.0); $loop_count = 0 loop13: -sleep 100 +sleep 50 sql select * from test.streamt4 order by c1, c2, c3; $loop_count = $loop_count + 1 @@ -534,7 +534,7 @@ sql insert into t1 values(1648791213000,1,2,3,1.0); $loop_count = 0 loop14: -sleep 100 +sleep 50 sql select * from test.streamt4 order by c1, c2, c3; $loop_count = $loop_count + 1 diff --git a/tests/script/tsim/stream/partitionbyColumn1.sim b/tests/script/tsim/stream/partitionbyColumn1.sim index 7f5c53ebe3..1742d52cf0 100644 --- a/tests/script/tsim/stream/partitionbyColumn1.sim +++ b/tests/script/tsim/stream/partitionbyColumn1.sim @@ -24,11 +24,11 @@ sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); $loop_count = 0 loop0: -sleep 300 +sleep 50 sql select * from streamt order by c1, c4, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -45,12 +45,14 @@ endi sql insert into t1 values(1648791213000,1,2,3,1.0); +$loop_count = 0 + loop1: -sleep 300 +sleep 50 sql select * from streamt order by c1, c4, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -66,12 +68,14 @@ endi sql insert into t1 values(1648791213000,2,2,3,1.0); +$loop_count = 0 + loop2: -sleep 300 +sleep 50 sql select * from streamt order by c1, c4, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -90,12 +94,14 @@ sql insert into t1 values(1648791213001,2,2,3,1.0); sql insert into t1 values(1648791213002,2,2,3,1.0); sql insert into t1 values(1648791213002,1,2,3,1.0); +$loop_count = 0 + loop3: -sleep 300 +sleep 50 sql select * from streamt order by c1, c4, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -125,12 +131,14 @@ sql insert into t1 values(1648791223002,3,2,3,1.0); sql insert into t1 values(1648791223003,3,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); +$loop_count = 0 + loop4: -sleep 300 +sleep 50 sql select * from streamt order by c1, c4, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -199,11 +207,11 @@ sql insert into t1 values(1648791213001,1,2,3,2.0); $loop_count = 0 loop5: -sleep 300 +sleep 50 sql select * from streamt1 order by c1, c4, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -217,12 +225,14 @@ sql insert into t1 values(1648791223001,1,2,5,2.0); sql insert into t1 values(1648791223002,1,2,5,2.0); sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0); +$loop_count = 0 + loop6: -sleep 300 +sleep 50 sql select * from streamt1 order by c1, c4, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -282,11 +292,11 @@ sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); $loop_count = 0 loop7: -sleep 300 +sleep 50 sql select * from test.streamt2 order by c1, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -303,12 +313,14 @@ endi sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t2 values(1648791213000,1,2,3,1.0); +$loop_count = 0 + loop8: -sleep 300 +sleep 50 sql select * from test.streamt2 order by c1, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -324,12 +336,15 @@ endi sql insert into t1 values(1648791213000,2,2,3,1.0); sql insert into t2 values(1648791213000,2,2,3,1.0); + +$loop_count = 0 + loop9: -sleep 300 +sleep 50 sql select * from test.streamt2 order by c1, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -352,12 +367,14 @@ sql insert into t2 values(1648791213001,2,2,3,1.0); sql insert into t2 values(1648791213002,2,2,3,1.0); sql insert into t2 values(1648791213002,1,2,3,1.0); +$loop_count = 0 + loop10: -sleep 300 +sleep 50 sql select * from test.streamt2 order by c1, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -373,7 +390,7 @@ endi if $data11 != 2 thenloop4 print =====data11=$data11 - goto loop3 + goto loop10 endi if $data12 != 1 then @@ -392,12 +409,14 @@ sql insert into t2 values(1648791223002,3,2,3,1.0); sql insert into t2 values(1648791223003,3,2,3,1.0); sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); +$loop_count = 0 + loop11: -sleep 300 +sleep 50 sql select * from test.streamt2 order by c1, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -470,17 +489,17 @@ sql insert into t4 values(1648791213000,1,2,3,1.0); $loop_count = 0 loop13: -sleep 300 +sleep 50 sql select * from test.streamt4 order by c1, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi if $rows != 2 then print =====rows=$rows - goto loop14 + goto loop13 endi if $data01 != 1 then @@ -495,12 +514,12 @@ endi if $data11 != 3 then print =====data11=$data11 - goto loop11 + goto loop13 endi if $data12 != 2 then print =====data12=$data12 - goto loop11 + goto loop13 endi sql insert into t4 values(1648791213000,2,2,3,1.0); @@ -509,12 +528,14 @@ sql insert into t1 values(1648791233000,2,2,3,1.0); sql insert into t1 values(1648791213000,1,2,3,1.0); +$loop_count = 0 + loop14: -sleep 300 +sleep 50 sql select * from test.streamt4 order by c1, c2, c3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi diff --git a/tests/script/tsim/stream/partitionbyColumn2.sim b/tests/script/tsim/stream/partitionbyColumn2.sim index 3d9acbcac5..75d01b17ec 100644 --- a/tests/script/tsim/stream/partitionbyColumn2.sim +++ b/tests/script/tsim/stream/partitionbyColumn2.sim @@ -40,6 +40,8 @@ endi sql insert into t1 values(1648791213000,1,1,3,1.0); +$loop_count = 0 + loop1: sleep 300 sql select * from streamt order by c1, c4, c2, c3; @@ -61,6 +63,8 @@ endi sql insert into t1 values(1648791213000,2,1,3,1.0); +$loop_count = 0 + loop2: sleep 300 sql select * from streamt order by c1, c4, c2, c3; @@ -85,6 +89,8 @@ sql insert into t1 values(1648791213001,2,1,3,1.0); sql insert into t1 values(1648791213002,2,1,3,1.0); sql insert into t1 values(1648791213002,1,1,3,1.0); +$loop_count = 0 + loop3: sleep 300 sql select * from streamt order by c1, c4, c2, c3; @@ -120,6 +126,8 @@ sql insert into t1 values(1648791223002,3,2,3,1.0); sql insert into t1 values(1648791223003,3,2,3,1.0); sql insert into t1 values(1648791213001,1,1,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); +$loop_count = 0 + loop4: sleep 300 sql select * from streamt order by c1, c4, c2, c3; @@ -212,6 +220,8 @@ sql insert into t1 values(1648791223001,1,2,2,5); sql insert into t1 values(1648791223002,1,2,2,6); sql insert into t1 values(1648791213001,1,1,1,7) (1648791223002,1,1,2,8); +$loop_count = 0 + loop6: sleep 300 sql select * from streamt1 order by c1, c4, c2, c3; From e70db79d106d6ea3b68120d5effb8e1324b62bfd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 5 Sep 2022 17:31:41 +0800 Subject: [PATCH 34/34] refact: change sst to stt --- source/dnode/vnode/src/inc/tsdb.h | 38 ++-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 94 ++++----- source/dnode/vnode/src/tsdb/tsdbFS.c | 198 +++++++++--------- source/dnode/vnode/src/tsdb/tsdbFile.c | 42 ++-- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 62 +++--- source/dnode/vnode/src/tsdb/tsdbRead.c | 18 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 116 +++++----- source/dnode/vnode/src/tsdb/tsdbRetention.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 36 ++-- source/dnode/vnode/src/tsdb/tsdbUtil.c | 56 ++--- 10 files changed, 331 insertions(+), 331 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 91aacaa328..8234cdff02 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -43,14 +43,14 @@ typedef struct STbDataIter STbDataIter; typedef struct SMapData SMapData; typedef struct SBlockIdx SBlockIdx; typedef struct SDataBlk SDataBlk; -typedef struct SSstBlk SSstBlk; +typedef struct SSttBlk SSttBlk; typedef struct SColData SColData; typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SBlockData SBlockData; typedef struct SDelFile SDelFile; typedef struct SHeadFile SHeadFile; typedef struct SDataFile SDataFile; -typedef struct SSstFile SSstFile; +typedef struct SSttFile SSttFile; typedef struct SSmaFile SSmaFile; typedef struct SDFileSet SDFileSet; typedef struct SDataFWriter SDataFWriter; @@ -69,8 +69,8 @@ typedef struct SLDataIter SLDataIter; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 -#define TSDB_MAX_SST_FILE 16 -#define TSDB_DEFAULT_SST_FILE 8 +#define TSDB_MAX_STT_FILE 16 +#define TSDB_DEFAULT_STT_FILE 8 #define TSDB_FHDR_SIZE 512 #define TSDB_DEFAULT_PAGE_SIZE 4096 @@ -130,9 +130,9 @@ int32_t tPutDataBlk(uint8_t *p, void *ph); int32_t tGetDataBlk(uint8_t *p, void *ph); int32_t tDataBlkCmprFn(const void *p1, const void *p2); bool tDataBlkHasSma(SDataBlk *pDataBlk); -// SSstBlk -int32_t tPutSstBlk(uint8_t *p, void *ph); -int32_t tGetSstBlk(uint8_t *p, void *ph); +// SSttBlk +int32_t tPutSttBlk(uint8_t *p, void *ph); +int32_t tGetSttBlk(uint8_t *p, void *ph); // SBlockIdx int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph); @@ -228,7 +228,7 @@ bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype); int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile); int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile); -int32_t tPutSstFile(uint8_t *p, SSstFile *pSstFile); +int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile); int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile); int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile); int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile); @@ -237,7 +237,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet); void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]); void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]); -void tsdbSstFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSstFile *pSstF, char fname[]); +void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]); void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]); // SDelFile void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]); @@ -263,7 +263,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx); -int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk); +int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast); @@ -273,10 +273,10 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); -int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk); +int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk); int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData); -int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData); +int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData); // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); @@ -448,7 +448,7 @@ struct SDataBlk { SSmaInfo smaInfo; }; -struct SSstBlk { +struct SSttBlk { int64_t suid; int64_t minUid; int64_t maxUid; @@ -550,7 +550,7 @@ struct SDataFile { int64_t size; }; -struct SSstFile { +struct SSttFile { volatile int32_t nRef; int64_t commitID; @@ -571,8 +571,8 @@ struct SDFileSet { SHeadFile *pHeadF; SDataFile *pDataF; SSmaFile *pSmaF; - uint8_t nSstF; - SSstFile *aSstF[TSDB_MAX_SST_FILE]; + uint8_t nSttF; + SSttFile *aSttF[TSDB_MAX_STT_FILE]; }; struct SRowIter { @@ -617,12 +617,12 @@ struct SDataFWriter { STsdbFD *pHeadFD; STsdbFD *pDataFD; STsdbFD *pSmaFD; - STsdbFD *pSstFD; + STsdbFD *pSttFD; SHeadFile fHead; SDataFile fData; SSmaFile fSma; - SSstFile fSst[TSDB_MAX_SST_FILE]; + SSttFile fStt[TSDB_MAX_STT_FILE]; uint8_t *aBuf[4]; }; @@ -633,7 +633,7 @@ struct SDataFReader { STsdbFD *pHeadFD; STsdbFD *pDataFD; STsdbFD *pSmaFD; - STsdbFD *aSstFD[TSDB_MAX_SST_FILE]; + STsdbFD *aSttFD[TSDB_MAX_STT_FILE]; uint8_t *aBuf[3]; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 6f7a78ee46..fb06203605 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -32,12 +32,12 @@ typedef struct { STbDataIter iter; }; // memory data iter struct { - int32_t iSst; - SArray *aSstBlk; - int32_t iSstBlk; + int32_t iStt; + SArray *aSttBlk; + int32_t iSttBlk; SBlockData bData; int32_t iRow; - }; // sst file data iter + }; // stt file data iter }; } SDataIter; @@ -71,13 +71,13 @@ typedef struct { SDataIter *pIter; SRBTree rbt; SDataIter dataIter; - SDataIter aDataIter[TSDB_MAX_SST_FILE]; + SDataIter aDataIter[TSDB_MAX_STT_FILE]; int8_t toLastOnly; }; struct { SDataFWriter *pWriter; SArray *aBlockIdx; // SArray - SArray *aSstBlk; // SArray + SArray *aSttBlk; // SArray SMapData mBlock; // SMapData SBlockData bData; SBlockData bDatal; @@ -428,21 +428,21 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { pCommitter->toLastOnly = 0; SDataFReader *pReader = pCommitter->dReader.pReader; if (pReader) { - if (pReader->pSet->nSstF >= pCommitter->maxLast) { + if (pReader->pSet->nSttF >= pCommitter->maxLast) { int8_t iIter = 0; - for (int32_t iSst = 0; iSst < pReader->pSet->nSstF; iSst++) { + for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) { pIter = &pCommitter->aDataIter[iIter]; pIter->type = LAST_DATA_ITER; - pIter->iSst = iSst; + pIter->iStt = iStt; - code = tsdbReadSstBlk(pCommitter->dReader.pReader, iSst, pIter->aSstBlk); + code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk); if (code) goto _err; - if (taosArrayGetSize(pIter->aSstBlk) == 0) continue; + if (taosArrayGetSize(pIter->aSttBlk) == 0) continue; - pIter->iSstBlk = 0; - SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, 0); - code = tsdbReadSstBlock(pCommitter->dReader.pReader, iSst, pSstBlk, &pIter->bData); + pIter->iSttBlk = 0; + SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0); + code = tsdbReadSttBlock(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData); if (code) goto _err; pIter->iRow = 0; @@ -454,9 +454,9 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { iIter++; } } else { - for (int32_t iSst = 0; iSst < pReader->pSet->nSstF; iSst++) { - SSstFile *pSstFile = pReader->pSet->aSstF[iSst]; - if (pSstFile->size > pSstFile->offset) { + for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) { + SSttFile *pSttFile = pReader->pSet->aSttF[iStt]; + if (pSttFile->size > pSttFile->offset) { pCommitter->toLastOnly = 1; break; } @@ -512,34 +512,34 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { SHeadFile fHead = {.commitID = pCommitter->commitID}; SDataFile fData = {.commitID = pCommitter->commitID}; SSmaFile fSma = {.commitID = pCommitter->commitID}; - SSstFile fSst = {.commitID = pCommitter->commitID}; + SSttFile fStt = {.commitID = pCommitter->commitID}; SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma}; if (pRSet) { - ASSERT(pRSet->nSstF <= pCommitter->maxLast); + ASSERT(pRSet->nSttF <= pCommitter->maxLast); fData = *pRSet->pDataF; fSma = *pRSet->pSmaF; wSet.diskId = pRSet->diskId; - if (pRSet->nSstF < pCommitter->maxLast) { - for (int32_t iSst = 0; iSst < pRSet->nSstF; iSst++) { - wSet.aSstF[iSst] = pRSet->aSstF[iSst]; + if (pRSet->nSttF < pCommitter->maxLast) { + for (int32_t iStt = 0; iStt < pRSet->nSttF; iStt++) { + wSet.aSttF[iStt] = pRSet->aSttF[iStt]; } - wSet.nSstF = pRSet->nSstF + 1; + wSet.nSttF = pRSet->nSttF + 1; } else { - wSet.nSstF = 1; + wSet.nSttF = 1; } } else { SDiskID did = {0}; tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did); tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); wSet.diskId = did; - wSet.nSstF = 1; + wSet.nSttF = 1; } - wSet.aSstF[wSet.nSstF - 1] = &fSst; + wSet.aSttF[wSet.nSttF - 1] = &fStt; code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); if (code) goto _err; taosArrayClear(pCommitter->dWriter.aBlockIdx); - taosArrayClear(pCommitter->dWriter.aSstBlk); + taosArrayClear(pCommitter->dWriter.aSttBlk); tMapDataReset(&pCommitter->dWriter.mBlock); tBlockDataReset(&pCommitter->dWriter.bData); tBlockDataReset(&pCommitter->dWriter.bDatal); @@ -610,7 +610,7 @@ _err: static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { int32_t code = 0; - SSstBlk blockL; + SSttBlk blockL; SBlockData *pBlockData = &pCommitter->dWriter.bDatal; ASSERT(pBlockData->nRow > 0); @@ -635,8 +635,8 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1); if (code) goto _err; - // push SSstBlk - if (taosArrayPush(pCommitter->dWriter.aSstBlk, &blockL) == NULL) { + // push SSttBlk + if (taosArrayPush(pCommitter->dWriter.aSttBlk, &blockL) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -658,8 +658,8 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); if (code) goto _err; - // write aSstBlk - code = tsdbWriteSstBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSstBlk); + // write aSttBlk + code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk); if (code) goto _err; // update file header @@ -757,7 +757,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; - pCommitter->maxLast = TSDB_DEFAULT_SST_FILE; // TODO: make it as a config + pCommitter->maxLast = TSDB_DEFAULT_STT_FILE; // TODO: make it as a config pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); if (pCommitter->aTbDataP == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -787,10 +787,10 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { if (code) goto _exit; // merger - for (int32_t iSst = 0; iSst < TSDB_MAX_SST_FILE; iSst++) { - SDataIter *pIter = &pCommitter->aDataIter[iSst]; - pIter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk)); - if (pIter->aSstBlk == NULL) { + for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) { + SDataIter *pIter = &pCommitter->aDataIter[iStt]; + pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if (pIter->aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -806,8 +806,8 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { goto _exit; } - pCommitter->dWriter.aSstBlk = taosArrayInit(0, sizeof(SSstBlk)); - if (pCommitter->dWriter.aSstBlk == NULL) { + pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if (pCommitter->dWriter.aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -829,15 +829,15 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { tBlockDataDestroy(&pCommitter->dReader.bData, 1); // merger - for (int32_t iSst = 0; iSst < TSDB_MAX_SST_FILE; iSst++) { - SDataIter *pIter = &pCommitter->aDataIter[iSst]; - taosArrayDestroy(pIter->aSstBlk); + for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) { + SDataIter *pIter = &pCommitter->aDataIter[iStt]; + taosArrayDestroy(pIter->aSttBlk); tBlockDataDestroy(&pIter->bData, 1); } // writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); - taosArrayDestroy(pCommitter->dWriter.aSstBlk); + taosArrayDestroy(pCommitter->dWriter.aSttBlk); tMapDataClear(&pCommitter->dWriter.mBlock); tBlockDataDestroy(&pCommitter->dWriter.bData, 1); tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); @@ -1052,11 +1052,11 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; pIter->r.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); } else { - pIter->iSstBlk++; - if (pIter->iSstBlk < taosArrayGetSize(pIter->aSstBlk)) { - SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk); + pIter->iSttBlk++; + if (pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk)) { + SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); - code = tsdbReadSstBlock(pCommitter->dReader.pReader, pIter->iSst, pSstBlk, &pIter->bData); + code = tsdbReadSttBlock(pCommitter->dReader.pReader, pIter->iStt, pSttBlk, &pIter->bData); if (code) goto _exit; pIter->iRow = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 14bc1214a6..8ab733aa56 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -113,7 +113,7 @@ _err: // taosRemoveFile(fname); // } -// // sst +// // stt // if (isSameDisk && pFrom->pLastF->commitID == pTo->pLastF->commitID) { // if (pFrom->pLastF->size > pTo->pLastF->size) { // code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_LAST_FILE); @@ -143,7 +143,7 @@ _err: // tsdbDataFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pDataF, fname); // taosRemoveFile(fname); -// // sst +// // stt // tsdbLastFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pLastF, fname); // taosRemoveFile(fname); @@ -258,8 +258,8 @@ void tsdbFSDestroy(STsdbFS *pFS) { taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pDataF); taosMemoryFree(pSet->pSmaF); - for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { - taosMemoryFree(pSet->aSstF[iSst]); + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + taosMemoryFree(pSet->aSttF[iStt]); } } @@ -328,14 +328,14 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { if (code) goto _err; } - // sst =========== - for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { - tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname); + // stt =========== + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); if (taosStatFile(fname, &size, NULL)) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size != LOGIC_TO_FILE_SIZE(pSet->aSstF[iSst]->size, TSDB_DEFAULT_PAGE_SIZE)) { + if (size != LOGIC_TO_FILE_SIZE(pSet->aSttF[iStt]->size, TSDB_DEFAULT_PAGE_SIZE)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -519,10 +519,10 @@ int32_t tsdbFSClose(STsdb *pTsdb) { ASSERT(pSet->pSmaF->nRef == 1); taosMemoryFree(pSet->pSmaF); - // sst - for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { - ASSERT(pSet->aSstF[iSst]->nRef == 1); - taosMemoryFree(pSet->aSstF[iSst]); + // stt + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + ASSERT(pSet->aSttF[iStt]->nRef == 1); + taosMemoryFree(pSet->aSttF[iStt]); } } @@ -579,14 +579,14 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { } *fSet.pSmaF = *pSet->pSmaF; - // sst - for (fSet.nSstF = 0; fSet.nSstF < pSet->nSstF; fSet.nSstF++) { - fSet.aSstF[fSet.nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (fSet.aSstF[fSet.nSstF] == NULL) { + // stt + for (fSet.nSttF = 0; fSet.nSttF < pSet->nSttF; fSet.nSttF++) { + fSet.aSttF[fSet.nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (fSet.aSttF[fSet.nSttF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - *fSet.aSstF[fSet.nSstF] = *pSet->aSstF[fSet.nSstF]; + *fSet.aSttF[fSet.nSttF] = *pSet->aSttF[fSet.nSttF]; } if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { @@ -639,28 +639,28 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { *pDFileSet->pHeadF = *pSet->pHeadF; *pDFileSet->pDataF = *pSet->pDataF; *pDFileSet->pSmaF = *pSet->pSmaF; - // sst - if (pSet->nSstF > pDFileSet->nSstF) { - ASSERT(pSet->nSstF == pDFileSet->nSstF + 1); + // stt + if (pSet->nSttF > pDFileSet->nSttF) { + ASSERT(pSet->nSttF == pDFileSet->nSttF + 1); - pDFileSet->aSstF[pDFileSet->nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (pDFileSet->aSstF[pDFileSet->nSstF] == NULL) { + pDFileSet->aSttF[pDFileSet->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pDFileSet->aSttF[pDFileSet->nSttF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - *pDFileSet->aSstF[pDFileSet->nSstF] = *pSet->aSstF[pSet->nSstF - 1]; - pDFileSet->nSstF++; - } else if (pSet->nSstF < pDFileSet->nSstF) { - ASSERT(pSet->nSstF == 1); - for (int32_t iSst = 1; iSst < pDFileSet->nSstF; iSst++) { - taosMemoryFree(pDFileSet->aSstF[iSst]); + *pDFileSet->aSttF[pDFileSet->nSttF] = *pSet->aSttF[pSet->nSttF - 1]; + pDFileSet->nSttF++; + } else if (pSet->nSttF < pDFileSet->nSttF) { + ASSERT(pSet->nSttF == 1); + for (int32_t iStt = 1; iStt < pDFileSet->nSttF; iStt++) { + taosMemoryFree(pDFileSet->aSttF[iStt]); } - *pDFileSet->aSstF[0] = *pSet->aSstF[0]; - pDFileSet->nSstF = 1; + *pDFileSet->aSttF[0] = *pSet->aSttF[0]; + pDFileSet->nSttF = 1; } else { - for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { - *pDFileSet->aSstF[iSst] = *pSet->aSstF[iSst]; + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + *pDFileSet->aSttF[iStt] = *pSet->aSttF[iStt]; } } @@ -668,8 +668,8 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } } - ASSERT(pSet->nSstF == 1); - SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nSstF = 1}; + ASSERT(pSet->nSttF == 1); + SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nSttF = 1}; // head fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); @@ -695,13 +695,13 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } *fSet.pSmaF = *pSet->pSmaF; - // sst - fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (fSet.aSstF[0] == NULL) { + // stt + fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (fSet.aSttF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - *fSet.aSstF[0] = *pSet->aSstF[0]; + *fSet.aSttF[0] = *pSet->aSttF[0]; if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -869,81 +869,81 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { pSetOld->pSmaF->size = pSetNew->pSmaF->size; } - // sst + // stt if (sameDisk) { - if (pSetNew->nSstF > pSetOld->nSstF) { - ASSERT(pSetNew->nSstF = pSetOld->nSstF + 1); - pSetOld->aSstF[pSetOld->nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (pSetOld->aSstF[pSetOld->nSstF] == NULL) { + if (pSetNew->nSttF > pSetOld->nSttF) { + ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1); + pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[pSetOld->nSttF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *pSetOld->aSstF[pSetOld->nSstF] = *pSetNew->aSstF[pSetOld->nSstF]; - pSetOld->aSstF[pSetOld->nSstF]->nRef = 1; - pSetOld->nSstF++; - } else if (pSetNew->nSstF < pSetOld->nSstF) { - ASSERT(pSetNew->nSstF == 1); - for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) { - SSstFile *pSstFile = pSetOld->aSstF[iSst]; - nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); + *pSetOld->aSttF[pSetOld->nSttF] = *pSetNew->aSttF[pSetOld->nSttF]; + pSetOld->aSttF[pSetOld->nSttF]->nRef = 1; + pSetOld->nSttF++; + } else if (pSetNew->nSttF < pSetOld->nSttF) { + ASSERT(pSetNew->nSttF == 1); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); if (nRef == 0) { - tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); taosRemoveFile(fname); - taosMemoryFree(pSstFile); + taosMemoryFree(pSttFile); } - pSetOld->aSstF[iSst] = NULL; + pSetOld->aSttF[iStt] = NULL; } - pSetOld->nSstF = 1; - pSetOld->aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (pSetOld->aSstF[0] == NULL) { + pSetOld->nSttF = 1; + pSetOld->aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *pSetOld->aSstF[0] = *pSetNew->aSstF[0]; - pSetOld->aSstF[0]->nRef = 1; + *pSetOld->aSttF[0] = *pSetNew->aSttF[0]; + pSetOld->aSttF[0]->nRef = 1; } else { - for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) { - if (pSetOld->aSstF[iSst]->commitID != pSetNew->aSstF[iSst]->commitID) { - SSstFile *pSstFile = pSetOld->aSstF[iSst]; - nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + if (pSetOld->aSttF[iStt]->commitID != pSetNew->aSttF[iStt]->commitID) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); if (nRef == 0) { - tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); taosRemoveFile(fname); - taosMemoryFree(pSstFile); + taosMemoryFree(pSttFile); } - pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (pSetOld->aSstF[iSst] == NULL) { + pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[iStt] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst]; - pSetOld->aSstF[iSst]->nRef = 1; + *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; + pSetOld->aSttF[iStt]->nRef = 1; } else { - ASSERT(pSetOld->aSstF[iSst]->size == pSetOld->aSstF[iSst]->size); - ASSERT(pSetOld->aSstF[iSst]->offset == pSetOld->aSstF[iSst]->offset); + ASSERT(pSetOld->aSttF[iStt]->size == pSetOld->aSttF[iStt]->size); + ASSERT(pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset); } } } } else { - ASSERT(pSetOld->nSstF == pSetNew->nSstF); - for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) { - SSstFile *pSstFile = pSetOld->aSstF[iSst]; - nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); + ASSERT(pSetOld->nSttF == pSetNew->nSttF); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); if (nRef == 0) { - tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); taosRemoveFile(fname); - taosMemoryFree(pSstFile); + taosMemoryFree(pSttFile); } - pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (pSetOld->aSstF[iSst] == NULL) { + pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[iStt] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst]; - pSetOld->aSstF[iSst]->nRef = 1; + *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; + pSetOld->aSttF[iStt]->nRef = 1; } } @@ -977,12 +977,12 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { taosMemoryFree(pSetOld->pSmaF); } - for (int8_t iSst = 0; iSst < pSetOld->nSstF; iSst++) { - nRef = atomic_sub_fetch_32(&pSetOld->aSstF[iSst]->nRef, 1); + for (int8_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + nRef = atomic_sub_fetch_32(&pSetOld->aSttF[iStt]->nRef, 1); if (nRef == 0) { - tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSstF[iSst], fname); + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSttF[iStt], fname); taosRemoveFile(fname); - taosMemoryFree(pSetOld->aSstF[iSst]); + taosMemoryFree(pSetOld->aSttF[iStt]); } } @@ -990,7 +990,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { continue; _add_new: - fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSstF = 1}; + fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSttF = 1}; // head fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); @@ -1019,15 +1019,15 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { *fSet.pSmaF = *pSetNew->pSmaF; fSet.pSmaF->nRef = 1; - // sst - ASSERT(pSetNew->nSstF == 1); - fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (fSet.aSstF[0] == NULL) { + // stt + ASSERT(pSetNew->nSttF == 1); + fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (fSet.aSttF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *fSet.aSstF[0] = *pSetNew->aSstF[0]; - fSet.aSstF[0]->nRef = 1; + *fSet.aSttF[0] = *pSetNew->aSttF[0]; + fSet.aSttF[0]->nRef = 1; if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1075,8 +1075,8 @@ int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1); ASSERT(nRef > 0); - for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { - nRef = atomic_fetch_add_32(&pSet->aSstF[iSst]->nRef, 1); + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + nRef = atomic_fetch_add_32(&pSet->aSttF[iStt]->nRef, 1); ASSERT(nRef > 0); } @@ -1134,14 +1134,14 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { taosMemoryFree(pSet->pSmaF); } - // sst - for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { - nRef = atomic_sub_fetch_32(&pSet->aSstF[iSst]->nRef, 1); + // stt + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + nRef = atomic_sub_fetch_32(&pSet->aSttF[iStt]->nRef, 1); ASSERT(nRef >= 0); if (nRef == 0) { - tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname); + tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); taosRemoveFile(fname); - taosMemoryFree(pSet->aSstF[iSst]); + taosMemoryFree(pSet->aSttF[iStt]); /* code */ } } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 2a7966e423..619fc17f5b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -53,22 +53,22 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) { return n; } -int32_t tPutSstFile(uint8_t *p, SSstFile *pSstFile) { +int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile) { int32_t n = 0; - n += tPutI64v(p ? p + n : p, pSstFile->commitID); - n += tPutI64v(p ? p + n : p, pSstFile->size); - n += tPutI64v(p ? p + n : p, pSstFile->offset); + n += tPutI64v(p ? p + n : p, pSttFile->commitID); + n += tPutI64v(p ? p + n : p, pSttFile->size); + n += tPutI64v(p ? p + n : p, pSttFile->offset); return n; } -static int32_t tGetSstFile(uint8_t *p, SSstFile *pSstFile) { +static int32_t tGetSttFile(uint8_t *p, SSttFile *pSttFile) { int32_t n = 0; - n += tGetI64v(p + n, &pSstFile->commitID); - n += tGetI64v(p + n, &pSstFile->size); - n += tGetI64v(p + n, &pSstFile->offset); + n += tGetI64v(p + n, &pSttFile->commitID); + n += tGetI64v(p + n, &pSttFile->size); + n += tGetI64v(p + n, &pSttFile->offset); return n; } @@ -102,9 +102,9 @@ void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pDataF->commitID, ".data"); } -void tsdbSstFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSstFile *pSstF, char fname[]) { +void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]) { snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did), - TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pSstF->commitID, ".sst"); + TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pSttF->commitID, ".stt"); } void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) { @@ -194,10 +194,10 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { n += tPutDataFile(p ? p + n : p, pSet->pDataF); n += tPutSmaFile(p ? p + n : p, pSet->pSmaF); - // sst - n += tPutU8(p ? p + n : p, pSet->nSstF); - for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { - n += tPutSstFile(p ? p + n : p, pSet->aSstF[iSst]); + // stt + n += tPutU8(p ? p + n : p, pSet->nSttF); + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + n += tPutSttFile(p ? p + n : p, pSet->aSttF[iStt]); } return n; @@ -234,15 +234,15 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { pSet->pSmaF->nRef = 1; n += tGetSmaFile(p + n, pSet->pSmaF); - // sst - n += tGetU8(p + n, &pSet->nSstF); - for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { - pSet->aSstF[iSst] = (SSstFile *)taosMemoryCalloc(1, sizeof(SSstFile)); - if (pSet->aSstF[iSst] == NULL) { + // stt + n += tGetU8(p + n, &pSet->nSttF); + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile)); + if (pSet->aSttF[iStt] == NULL) { return -1; } - pSet->aSstF[iSst]->nRef = 1; - n += tGetSstFile(p + n, pSet->aSstF[iSst]); + pSet->aSttF[iStt]->nRef = 1; + n += tGetSttFile(p + n, pSet->aSttF[iStt]); } return n; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 171e32007a..b7eedc20ff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -18,12 +18,12 @@ // SLDataIter ================================================= struct SLDataIter { SRBTreeNode node; - SSstBlk *pSstBlk; + SSttBlk *pSttBlk; SDataFReader *pReader; - int32_t iSst; + int32_t iStt; int8_t backward; - SArray *aSstBlk; - int32_t iSstBlk; + SArray *aSttBlk; + int32_t iSttBlk; SBlockData bData[2]; int32_t loadIndex; int32_t iRow; @@ -40,7 +40,7 @@ static SBlockData *getNextBlock(SLDataIter *pIter) { return getCurrentBlock(pIter); } -int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iSst, int8_t backward, uint64_t uid, +int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange) { int32_t code = 0; *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); @@ -53,10 +53,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t (*pIter)->timeWindow = *pTimeWindow; (*pIter)->verRange = *pRange; (*pIter)->pReader = pReader; - (*pIter)->iSst = iSst; + (*pIter)->iStt = iStt; (*pIter)->backward = backward; - (*pIter)->aSstBlk = taosArrayInit(0, sizeof(SSstBlk)); - if ((*pIter)->aSstBlk == NULL) { + (*pIter)->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if ((*pIter)->aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -71,18 +71,18 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t goto _exit; } - code = tsdbReadSstBlk(pReader, iSst, (*pIter)->aSstBlk); + code = tsdbReadSttBlk(pReader, iStt, (*pIter)->aSttBlk); if (code) { goto _exit; } - size_t size = taosArrayGetSize((*pIter)->aSstBlk); + size_t size = taosArrayGetSize((*pIter)->aSttBlk); // find the start block int32_t index = -1; if (!backward) { // asc for (int32_t i = 0; i < size; ++i) { - SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i); + SSttBlk *p = taosArrayGet((*pIter)->aSttBlk, i); if (p->minUid <= uid && p->maxUid >= uid) { index = i; break; @@ -90,7 +90,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t } } else { // desc for (int32_t i = size - 1; i >= 0; --i) { - SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i); + SSttBlk *p = taosArrayGet((*pIter)->aSttBlk, i); if (p->minUid <= uid && p->maxUid >= uid) { index = i; break; @@ -98,9 +98,9 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t } } - (*pIter)->iSstBlk = index; + (*pIter)->iSttBlk = index; if (index != -1) { - (*pIter)->pSstBlk = taosArrayGet((*pIter)->aSstBlk, (*pIter)->iSstBlk); + (*pIter)->pSttBlk = taosArrayGet((*pIter)->aSttBlk, (*pIter)->iSttBlk); } _exit: @@ -110,18 +110,18 @@ _exit: void tLDataIterClose(SLDataIter *pIter) { tBlockDataDestroy(&pIter->bData[0], 1); tBlockDataDestroy(&pIter->bData[1], 1); - taosArrayDestroy(pIter->aSstBlk); + taosArrayDestroy(pIter->aSttBlk); taosMemoryFree(pIter); } void tLDataIterNextBlock(SLDataIter *pIter) { int32_t step = pIter->backward ? -1 : 1; - pIter->iSstBlk += step; + pIter->iSttBlk += step; int32_t index = -1; - size_t size = taosArrayGetSize(pIter->aSstBlk); - for (int32_t i = pIter->iSstBlk; i < size && i >= 0; i += step) { - SSstBlk *p = taosArrayGet(pIter->aSstBlk, i); + size_t size = taosArrayGetSize(pIter->aSttBlk); + for (int32_t i = pIter->iSttBlk; i < size && i >= 0; i += step) { + SSttBlk *p = taosArrayGet(pIter->aSttBlk, i); if ((!pIter->backward) && p->minUid > pIter->uid) { break; } @@ -137,9 +137,9 @@ void tLDataIterNextBlock(SLDataIter *pIter) { } if (index == -1) { - pIter->pSstBlk = NULL; + pIter->pSttBlk = NULL; } else { - pIter->pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk); + pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); } } @@ -212,16 +212,16 @@ bool tLDataIterNextRow(SLDataIter *pIter) { int32_t step = pIter->backward ? -1 : 1; // no qualified last file block in current file, no need to fetch row - if (pIter->pSstBlk == NULL) { + if (pIter->pSttBlk == NULL) { return false; } - int32_t iBlockL = pIter->iSstBlk; + int32_t iBlockL = pIter->iSttBlk; SBlockData *pBlockData = getCurrentBlock(pIter); - if (pBlockData->nRow == 0 && pIter->pSstBlk != NULL) { // current block not loaded yet + if (pBlockData->nRow == 0 && pIter->pSttBlk != NULL) { // current block not loaded yet pBlockData = getNextBlock(pIter); - code = tsdbReadSstBlock(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData); + code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlockData); if (code != TSDB_CODE_SUCCESS) { goto _exit; } @@ -236,16 +236,16 @@ bool tLDataIterNextRow(SLDataIter *pIter) { if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) { tLDataIterNextBlock(pIter); - if (pIter->pSstBlk == NULL) { // no more data + if (pIter->pSttBlk == NULL) { // no more data goto _exit; } } else { break; } - if (iBlockL != pIter->iSstBlk) { + if (iBlockL != pIter->iSttBlk) { pBlockData = getNextBlock(pIter); - code = tsdbReadSstBlock(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData); + code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlockData); if (code) { goto _exit; } @@ -262,7 +262,7 @@ _exit: terrno = code; } - return (code == TSDB_CODE_SUCCESS) && (pIter->pSstBlk != NULL); + return (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); } SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } @@ -302,8 +302,8 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); int32_t code = TSDB_CODE_OUT_OF_MEMORY; - struct SLDataIter *pIterList[TSDB_DEFAULT_SST_FILE] = {0}; - for (int32_t i = 0; i < pFReader->pSet->nSstF; ++i) { // open all last file + struct SLDataIter *pIterList[TSDB_DEFAULT_STT_FILE] = {0}; + for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); if (code != TSDB_CODE_SUCCESS) { goto _end; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 1828f1e40e..d895056f9e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -621,7 +621,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN } } - pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nSstF; + pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nSttF; int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks; double el = (taosGetTimestampUs() - st) / 1000.0; @@ -1163,7 +1163,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc bool overlapWithlastBlock = false; #if 0 if (taosArrayGetSize(pLastBlockReader->pSstBlk) > 0 && (pLastBlockReader->currentBlockIndex != -1)) { - SSstBlk* pSstBlk = taosArrayGet(pLastBlockReader->pSstBlk, pLastBlockReader->currentBlockIndex); + SSttBlk* pSstBlk = taosArrayGet(pLastBlockReader->pSstBlk, pLastBlockReader->currentBlockIndex); overlapWithlastBlock = !(pBlock->maxKey.ts < pSstBlk->minKey || pBlock->minKey.ts > pSstBlk->maxKey); } #endif @@ -1380,7 +1380,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, bool mergeBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; // SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); STSRow* pTSRow = NULL; SRowMerger merge = {0}; @@ -1779,7 +1779,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL); tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 " %s", + "-%" PRId64 " %s", pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, d->minKey, d->maxKey, pReader->idStr); } else { tsdbError("%p uid:%" PRId64 ", failed to create iterator for imem, code:%s, %s", pReader, pBlockScanInfo->uid, @@ -1800,7 +1800,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea pBlockScanInfo->iiter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iiter.iter) != NULL); tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 " %s", + "-%" PRId64 " %s", pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, di->minKey, di->maxKey, pReader->idStr); } else { tsdbError("%p uid:%" PRId64 ", failed to create iterator for mem, code:%s, %s", pReader, pBlockScanInfo->uid, @@ -1850,7 +1850,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { - while(1) { + while (1) { bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); if (!hasVal) { return false; @@ -2166,7 +2166,7 @@ _err: } static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { - TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; + TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); if (pRow != NULL) { key = TSDBROW_KEY(pRow); @@ -2204,7 +2204,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return code; } - if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nSstF > 0) { + if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nSttF > 0) { code = doLoadFileBlock(pReader, pIndexList, pBlockNum); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); @@ -2314,7 +2314,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { while (1) { // load the last data block of current table STableBlockScanInfo* pScanInfo = pStatus->pTableIter; - bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); if (!hasVal) { bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); if (!hasNexTable) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 25daec76c6..128bfe37da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -226,13 +226,13 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS .pHeadF = &pWriter->fHead, .pDataF = &pWriter->fData, .pSmaF = &pWriter->fSma, - .nSstF = pSet->nSstF}; + .nSttF = pSet->nSttF}; pWriter->fHead = *pSet->pHeadF; pWriter->fData = *pSet->pDataF; pWriter->fSma = *pSet->pSmaF; - for (int8_t iSst = 0; iSst < pSet->nSstF; iSst++) { - pWriter->wSet.aSstF[iSst] = &pWriter->fSst[iSst]; - pWriter->fSst[iSst] = *pSet->aSstF[iSst]; + for (int8_t iStt = 0; iStt < pSet->nSttF; iStt++) { + pWriter->wSet.aSttF[iStt] = &pWriter->fStt[iStt]; + pWriter->fStt[iStt] = *pSet->aSttF[iStt]; } // head @@ -276,15 +276,15 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS pWriter->fSma.size += TSDB_FHDR_SIZE; } - // sst - ASSERT(pWriter->fSst[pSet->nSstF - 1].size == 0); + // stt + ASSERT(pWriter->fStt[pSet->nSttF - 1].size == 0); flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - tsdbSstFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSst[pSet->nSstF - 1], fname); - code = tsdbOpenFile(fname, szPage, flag, &pWriter->pSstFD); + tsdbSttFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fStt[pSet->nSttF - 1], fname); + code = tsdbOpenFile(fname, szPage, flag, &pWriter->pSttFD); if (code) goto _err; - code = tsdbWriteFile(pWriter->pSstFD, 0, hdr, TSDB_FHDR_SIZE); + code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE); if (code) goto _err; - pWriter->fSst[pWriter->wSet.nSstF - 1].size += TSDB_FHDR_SIZE; + pWriter->fStt[pWriter->wSet.nSttF - 1].size += TSDB_FHDR_SIZE; *ppWriter = pWriter; return code; @@ -312,14 +312,14 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { code = tsdbFsyncFile((*ppWriter)->pSmaFD); if (code) goto _err; - code = tsdbFsyncFile((*ppWriter)->pSstFD); + code = tsdbFsyncFile((*ppWriter)->pSttFD); if (code) goto _err; } tsdbCloseFile(&(*ppWriter)->pHeadFD); tsdbCloseFile(&(*ppWriter)->pDataFD); tsdbCloseFile(&(*ppWriter)->pSmaFD); - tsdbCloseFile(&(*ppWriter)->pSstFD); + tsdbCloseFile(&(*ppWriter)->pSttFD); for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { tFree((*ppWriter)->aBuf[iBuf]); @@ -357,10 +357,10 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE); if (code) goto _err; - // sst ============== + // stt ============== memset(hdr, 0, TSDB_FHDR_SIZE); - tPutSstFile(hdr, &pWriter->fSst[pWriter->wSet.nSstF - 1]); - code = tsdbWriteFile(pWriter->pSstFD, 0, hdr, TSDB_FHDR_SIZE); + tPutSttFile(hdr, &pWriter->fStt[pWriter->wSet.nSttF - 1]); + code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE); if (code) goto _err; return code; @@ -454,22 +454,22 @@ _err: return code; } -int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { +int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) { int32_t code = 0; - SSstFile *pSstFile = &pWriter->fSst[pWriter->wSet.nSstF - 1]; + SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1]; int64_t size; int64_t n; // check - if (taosArrayGetSize(aSstBlk) == 0) { - pSstFile->offset = pSstFile->size; + if (taosArrayGetSize(aSttBlk) == 0) { + pSttFile->offset = pSttFile->size; goto _exit; } // size size = 0; - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) { - size += tPutSstBlk(NULL, taosArrayGet(aSstBlk, iBlockL)); + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) { + size += tPutSttBlk(NULL, taosArrayGet(aSttBlk, iBlockL)); } // alloc @@ -478,21 +478,21 @@ int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { // encode n = 0; - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) { - n += tPutSstBlk(pWriter->aBuf[0] + n, taosArrayGet(aSstBlk, iBlockL)); + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) { + n += tPutSttBlk(pWriter->aBuf[0] + n, taosArrayGet(aSttBlk, iBlockL)); } // write - code = tsdbWriteFile(pWriter->pSstFD, pSstFile->size, pWriter->aBuf[0], size); + code = tsdbWriteFile(pWriter->pSttFD, pSttFile->size, pWriter->aBuf[0], size); if (code) goto _err; // update - pSstFile->offset = pSstFile->size; - pSstFile->size += size; + pSttFile->offset = pSttFile->size; + pSttFile->size += size; _exit: - tsdbTrace("vgId:%d tsdb write sst block, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), - pSstFile->offset, size); + tsdbTrace("vgId:%d tsdb write stt block, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), + pSttFile->offset, size); return code; _err: @@ -546,7 +546,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock ASSERT(pBlockData->nRow > 0); if (toLast) { - pBlkInfo->offset = pWriter->fSst[pWriter->wSet.nSstF - 1].size; + pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size; } else { pBlkInfo->offset = pWriter->fData.size; } @@ -558,7 +558,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock if (code) goto _err; // write ================= - STsdbFD *pFD = toLast ? pWriter->pSstFD : pWriter->pDataFD; + STsdbFD *pFD = toLast ? pWriter->pSttFD : pWriter->pDataFD; pBlkInfo->szKey = aBufN[3] + aBufN[2]; pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; @@ -585,7 +585,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock // update info if (toLast) { - pWriter->fSst[pWriter->wSet.nSstF - 1].size += pBlkInfo->szBlock; + pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock; } else { pWriter->fData.size += pBlkInfo->szBlock; } @@ -664,9 +664,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { taosCloseFile(&pOutFD); taosCloseFile(&PInFD); - // sst - tsdbSstFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSstF[0], fNameFrom); - tsdbSstFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSstF[0], fNameTo); + // stt + tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[0], fNameFrom); + tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[0], fNameTo); pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { @@ -680,7 +680,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { goto _err; } - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aSstF[0]->size); + n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aSttF[0]->size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -750,10 +750,10 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD); if (code) goto _err; - // sst - for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { - tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname); - code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSstFD[iSst]); + // stt + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); + code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]); if (code) goto _err; } @@ -779,10 +779,10 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { // sma tsdbCloseFile(&(*ppReader)->pSmaFD); - // sst - for (int32_t iSst = 0; iSst < TSDB_MAX_SST_FILE; iSst++) { - if ((*ppReader)->aSstFD[iSst]) { - tsdbCloseFile(&(*ppReader)->aSstFD[iSst]); + // stt + for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) { + if ((*ppReader)->aSttFD[iStt]) { + tsdbCloseFile(&(*ppReader)->aSttFD[iStt]); } } @@ -835,13 +835,13 @@ _err: return code; } -int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) { +int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) { int32_t code = 0; - SSstFile *pSstFile = pReader->pSet->aSstF[iSst]; - int64_t offset = pSstFile->offset; - int64_t size = pSstFile->size - offset; + SSttFile *pSttFile = pReader->pSet->aSttF[iStt]; + int64_t offset = pSttFile->offset; + int64_t size = pSttFile->size - offset; - taosArrayClear(aSstBlk); + taosArrayClear(aSttBlk); if (size == 0) return code; // alloc @@ -849,16 +849,16 @@ int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) { if (code) goto _err; // read - code = tsdbReadFile(pReader->aSstFD[iSst], offset, pReader->aBuf[0], size); + code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size); if (code) goto _err; // decode int64_t n = 0; while (n < size) { - SSstBlk sstBlk; - n += tGetSstBlk(pReader->aBuf[0] + n, &sstBlk); + SSttBlk sttBlk; + n += tGetSttBlk(pReader->aBuf[0] + n, &sttBlk); - if (taosArrayPush(aSstBlk, &sstBlk) == NULL) { + if (taosArrayPush(aSttBlk, &sttBlk) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -868,7 +868,7 @@ int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) { return code; _err: - tsdbError("vgId:%d read sst blk failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d read stt blk failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } @@ -1107,25 +1107,25 @@ _err: return code; } -int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) { +int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData) { int32_t code = 0; // alloc - code = tRealloc(&pReader->aBuf[0], pSstBlk->bInfo.szBlock); + code = tRealloc(&pReader->aBuf[0], pSttBlk->bInfo.szBlock); if (code) goto _err; // read - code = tsdbReadFile(pReader->aSstFD[iSst], pSstBlk->bInfo.offset, pReader->aBuf[0], pSstBlk->bInfo.szBlock); + code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock); if (code) goto _err; // decmpr - code = tDecmprBlockData(pReader->aBuf[0], pSstBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); + code = tDecmprBlockData(pReader->aBuf[0], pSttBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); if (code) goto _err; return code; _err: - tsdbError("vgId:%d tsdb read sst block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d tsdb read stt block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index ee29538a81..d10613e719 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -60,7 +60,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { if (expLevel < 0) { taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->aSstF[0]); + taosMemoryFree(pSet->aSttF[0]); taosMemoryFree(pSet->pSmaF); taosArrayRemove(fs.aDFileSet, iSet); iSet--; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 8d19a2ffb8..9fc5639c5e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -27,9 +27,9 @@ struct STsdbSnapReader { int32_t fid; SDataFReader* pDataFReader; SArray* aBlockIdx; // SArray - SArray* aSstBlk; // SArray + SArray* aSstBlk; // SArray SBlockIdx* pBlockIdx; - SSstBlk* pSstBlk; + SSttBlk* pSstBlk; int32_t iBlockIdx; int32_t iBlockL; @@ -64,7 +64,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); if (code) goto _err; - code = tsdbReadSstBlk(pReader->pDataFReader, 0, pReader->aSstBlk); + code = tsdbReadSttBlk(pReader->pDataFReader, 0, pReader->aSstBlk); if (code) goto _err; // init @@ -87,7 +87,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { break; } - pReader->pSstBlk = (SSstBlk*)taosArrayGet(pReader->aSstBlk, pReader->iBlockL); + pReader->pSstBlk = (SSttBlk*)taosArrayGet(pReader->aSstBlk, pReader->iBlockL); if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) { // TODO break; @@ -151,7 +151,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { // next pReader->iBlockL++; if (pReader->iBlockL < taosArrayGetSize(pReader->aSstBlk)) { - pReader->pSstBlk = (SSstBlk*)taosArrayGetSize(pReader->aSstBlk); + pReader->pSstBlk = (SSttBlk*)taosArrayGetSize(pReader->aSstBlk); } else { pReader->pSstBlk = NULL; } @@ -298,7 +298,7 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pReader->aSstBlk = taosArrayInit(0, sizeof(SSstBlk)); + pReader->aSstBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pReader->aSstBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -431,7 +431,7 @@ struct STsdbSnapWriter { SBlockData* pBlockData; int32_t iRow; SBlockData bDataR; - SArray* aSstBlk; // SArray + SArray* aSstBlk; // SArray int32_t iBlockL; SBlockData lDataR; @@ -443,7 +443,7 @@ struct STsdbSnapWriter { SMapData mBlockW; // SMapData SArray* aBlockIdxW; // SArray - SArray* aBlockLW; // SArray + SArray* aBlockLW; // SArray // for del file SDelFReader* pDelFReader; @@ -845,7 +845,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { // write remain stuff if (taosArrayGetSize(pWriter->aBlockLW) > 0) { - code = tsdbWriteSstBlk(pWriter->pDataFWriter, pWriter->aBlockIdxW); + code = tsdbWriteSttBlk(pWriter->pDataFWriter, pWriter->aBlockIdxW); if (code) goto _err; } @@ -911,7 +911,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx); if (code) goto _err; - code = tsdbReadSstBlk(pWriter->pDataFReader, 0, pWriter->aSstBlk); + code = tsdbReadSttBlk(pWriter->pDataFReader, 0, pWriter->aSstBlk); if (code) goto _err; } else { ASSERT(pWriter->pDataFReader == NULL); @@ -931,25 +931,25 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 // write SHeadFile fHead; SDataFile fData; - SSstFile fLast; + SSttFile fLast; SSmaFile fSma; - SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aSstF[0] = &fLast, .pSmaF = &fSma}; + SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aSttF[0] = &fLast, .pSmaF = &fSma}; if (pSet) { wSet.diskId = pSet->diskId; wSet.fid = fid; - wSet.nSstF = 1; + wSet.nSttF = 1; fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fData = *pSet->pDataF; - fLast = (SSstFile){.commitID = pWriter->commitID, .size = 0}; + fLast = (SSttFile){.commitID = pWriter->commitID, .size = 0}; fSma = *pSet->pSmaF; } else { wSet.diskId = (SDiskID){.level = 0, .id = 0}; wSet.fid = fid; - wSet.nSstF = 1; + wSet.nSttF = 1; fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fData = (SDataFile){.commitID = pWriter->commitID, .size = 0}; - fLast = (SSstFile){.commitID = pWriter->commitID, .size = 0, .offset = 0}; + fLast = (SSttFile){.commitID = pWriter->commitID, .size = 0, .offset = 0}; fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0}; } @@ -1147,7 +1147,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = tBlockDataCreate(&pWriter->bDataR); if (code) goto _err; - pWriter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk)); + pWriter->aSstBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pWriter->aSstBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1161,7 +1161,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = tBlockDataCreate(&pWriter->bDataW); if (code) goto _err; - pWriter->aBlockLW = taosArrayInit(0, sizeof(SSstBlk)); + pWriter->aBlockLW = taosArrayInit(0, sizeof(SSttBlk)); if (pWriter->aBlockLW == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8509c0c759..caeca45e01 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -214,7 +214,7 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) { int32_t tCmprBlockL(void const *lhs, void const *rhs) { SBlockIdx *lBlockIdx = (SBlockIdx *)lhs; - SSstBlk *rBlockL = (SSstBlk *)rhs; + SSttBlk *rBlockL = (SSttBlk *)rhs; if (lBlockIdx->suid < rBlockL->suid) { return -1; @@ -311,41 +311,41 @@ bool tDataBlkHasSma(SDataBlk *pDataBlk) { return pDataBlk->smaInfo.size > 0; } -// SSstBlk ====================================================== -int32_t tPutSstBlk(uint8_t *p, void *ph) { +// SSttBlk ====================================================== +int32_t tPutSttBlk(uint8_t *p, void *ph) { int32_t n = 0; - SSstBlk *pSstBlk = (SSstBlk *)ph; + SSttBlk *pSttBlk = (SSttBlk *)ph; - n += tPutI64(p ? p + n : p, pSstBlk->suid); - n += tPutI64(p ? p + n : p, pSstBlk->minUid); - n += tPutI64(p ? p + n : p, pSstBlk->maxUid); - n += tPutI64v(p ? p + n : p, pSstBlk->minKey); - n += tPutI64v(p ? p + n : p, pSstBlk->maxKey); - n += tPutI64v(p ? p + n : p, pSstBlk->minVer); - n += tPutI64v(p ? p + n : p, pSstBlk->maxVer); - n += tPutI32v(p ? p + n : p, pSstBlk->nRow); - n += tPutI64v(p ? p + n : p, pSstBlk->bInfo.offset); - n += tPutI32v(p ? p + n : p, pSstBlk->bInfo.szBlock); - n += tPutI32v(p ? p + n : p, pSstBlk->bInfo.szKey); + n += tPutI64(p ? p + n : p, pSttBlk->suid); + n += tPutI64(p ? p + n : p, pSttBlk->minUid); + n += tPutI64(p ? p + n : p, pSttBlk->maxUid); + n += tPutI64v(p ? p + n : p, pSttBlk->minKey); + n += tPutI64v(p ? p + n : p, pSttBlk->maxKey); + n += tPutI64v(p ? p + n : p, pSttBlk->minVer); + n += tPutI64v(p ? p + n : p, pSttBlk->maxVer); + n += tPutI32v(p ? p + n : p, pSttBlk->nRow); + n += tPutI64v(p ? p + n : p, pSttBlk->bInfo.offset); + n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szBlock); + n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szKey); return n; } -int32_t tGetSstBlk(uint8_t *p, void *ph) { +int32_t tGetSttBlk(uint8_t *p, void *ph) { int32_t n = 0; - SSstBlk *pSstBlk = (SSstBlk *)ph; + SSttBlk *pSttBlk = (SSttBlk *)ph; - n += tGetI64(p + n, &pSstBlk->suid); - n += tGetI64(p + n, &pSstBlk->minUid); - n += tGetI64(p + n, &pSstBlk->maxUid); - n += tGetI64v(p + n, &pSstBlk->minKey); - n += tGetI64v(p + n, &pSstBlk->maxKey); - n += tGetI64v(p + n, &pSstBlk->minVer); - n += tGetI64v(p + n, &pSstBlk->maxVer); - n += tGetI32v(p + n, &pSstBlk->nRow); - n += tGetI64v(p + n, &pSstBlk->bInfo.offset); - n += tGetI32v(p + n, &pSstBlk->bInfo.szBlock); - n += tGetI32v(p + n, &pSstBlk->bInfo.szKey); + n += tGetI64(p + n, &pSttBlk->suid); + n += tGetI64(p + n, &pSttBlk->minUid); + n += tGetI64(p + n, &pSttBlk->maxUid); + n += tGetI64v(p + n, &pSttBlk->minKey); + n += tGetI64v(p + n, &pSttBlk->maxKey); + n += tGetI64v(p + n, &pSttBlk->minVer); + n += tGetI64v(p + n, &pSttBlk->maxVer); + n += tGetI32v(p + n, &pSttBlk->nRow); + n += tGetI64v(p + n, &pSttBlk->bInfo.offset); + n += tGetI32v(p + n, &pSttBlk->bInfo.szBlock); + n += tGetI32v(p + n, &pSttBlk->bInfo.szKey); return n; }