From ebd60a968c464cdf64cd392bb398366c15453107 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 1 Aug 2022 10:03:00 +0000 Subject: [PATCH 001/194] refact: tsdb last file optimize 1 --- source/dnode/vnode/src/inc/tsdb.h | 8 ++ source/dnode/vnode/src/tsdb/tsdbCommit.c | 75 +++++++++--- source/dnode/vnode/src/tsdb/tsdbFile.c | 2 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 109 ++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 50 ++++++++ 6 files changed, 231 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f1e980c026..8a6f473dc4 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -113,6 +113,9 @@ int32_t tPutBlock(uint8_t *p, void *ph); int32_t tGetBlock(uint8_t *p, void *ph); int32_t tBlockCmprFn(const void *p1, const void *p2); bool tBlockHasSma(SBlock *pBlock); +// SBlockL +int32_t tPutBlockL(uint8_t *p, void *ph); +int32_t tGetBlockL(uint8_t *p, void *ph); // SBlockIdx int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph); @@ -225,6 +228,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBuf, SBlockIdx *pBlockIdx); +int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2, SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg); @@ -233,6 +237,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf); +int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData, uint8_t **ppBuf); int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); @@ -416,6 +421,7 @@ struct SBlock { }; struct SBlockL { + int64_t suid; struct { int64_t uid; int64_t version; @@ -452,6 +458,7 @@ struct SColData { struct SBlockData { int32_t nRow; + int64_t *aUid; int64_t *aVersion; TSKEY *aTSKEY; SArray *aIdx; // SArray @@ -513,6 +520,7 @@ struct SHeadFile { int64_t commitID; int64_t size; int64_t offset; + int64_t loffset; }; struct SDataFile { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index e6db812865..dc1aa42dc6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -39,14 +39,18 @@ typedef struct { struct { SDataFReader *pReader; SArray *aBlockIdx; // SArray + SArray *aBlockL; // SArray SMapData mBlock; // SMapData, read from reader SBlockData bData; + SBlockData bDatal; } dReader; struct { SDataFWriter *pWriter; SArray *aBlockIdx; // SArray + SArray *aBlockL; // SArray SMapData mBlock; // SMapData SBlockData bData; + SBlockData bDatal; } dWriter; SSkmInfo skmTable; SSkmInfo skmRow; @@ -279,10 +283,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { // memory pCommitter->nextKey = TSKEY_MAX; - // old - taosArrayClear(pCommitter->dReader.aBlockIdx); - tMapDataReset(&pCommitter->dReader.mBlock); - tBlockDataReset(&pCommitter->dReader.bData); + // Reader pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &(SDFileSet){.fid = pCommitter->commitFid}, tDFileSetCmprFn, TD_EQ); if (pRSet) { @@ -291,22 +292,29 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx, NULL); if (code) goto _err; - } - // new + code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL, NULL); + if (code) goto _err; + + } else { + pCommitter->dReader.pReader = NULL; + taosArrayClear(pCommitter->dReader.aBlockIdx); + taosArrayClear(pCommitter->dReader.aBlockL); + } + tMapDataReset(&pCommitter->dReader.mBlock); + tBlockDataReset(&pCommitter->dReader.bData); + tBlockDataReset(&pCommitter->dReader.bDatal); + + // Writer SHeadFile fHead; SDataFile fData; SLastFile fLast; SSmaFile fSma; SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma}; - - taosArrayClear(pCommitter->dWriter.aBlockIdx); - tMapDataReset(&pCommitter->dWriter.mBlock); - tBlockDataReset(&pCommitter->dWriter.bData); if (pRSet) { wSet.diskId = pRSet->diskId; wSet.fid = pCommitter->commitFid; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0}; + fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0, .loffset = 0}; fData = *pRSet->pDataF; fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0}; fSma = *pRSet->pSmaF; @@ -319,7 +327,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { wSet.diskId = did; wSet.fid = pCommitter->commitFid; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0}; + fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0, .loffset = 0}; fData = (SDataFile){.commitID = pCommitter->commitID, .size = 0}; fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0}; fSma = (SSmaFile){.commitID = pCommitter->commitID, .size = 0}; @@ -327,6 +335,12 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); if (code) goto _err; + taosArrayClear(pCommitter->dWriter.aBlockIdx); + taosArrayClear(pCommitter->dWriter.aBlockL); + tMapDataReset(&pCommitter->dWriter.mBlock); + tBlockDataReset(&pCommitter->dWriter.bData); + tBlockDataReset(&pCommitter->dWriter.bDatal); + _exit: return code; @@ -861,7 +875,11 @@ _err: static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; - // write blockIdx + // write aBlockL + code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL, NULL); + if (code) goto _err; + + // write aBlockIdx code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx, NULL); if (code) goto _err; @@ -1001,14 +1019,15 @@ _err: static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { int32_t code = 0; + // Reader pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCommitter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pCommitter->dWriter.aBlockIdx == NULL) { + pCommitter->dReader.aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pCommitter->dReader.aBlockL == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -1016,20 +1035,46 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { code = tBlockDataInit(&pCommitter->dReader.bData); if (code) goto _exit; + code = tBlockDataInit(&pCommitter->dReader.bDatal); + if (code) goto _exit; + + // Writer + pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pCommitter->dWriter.aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pCommitter->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pCommitter->dWriter.aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + code = tBlockDataInit(&pCommitter->dWriter.bData); if (code) goto _exit; + code = tBlockDataInit(&pCommitter->dWriter.bDatal); + if (code) goto _exit; + _exit: return code; } static void tsdbCommitDataEnd(SCommitter *pCommitter) { + // Reader taosArrayDestroy(pCommitter->dReader.aBlockIdx); + taosArrayDestroy(pCommitter->dReader.aBlockL); tMapDataClear(&pCommitter->dReader.mBlock); tBlockDataClear(&pCommitter->dReader.bData, 1); + tBlockDataClear(&pCommitter->dReader.bDatal, 1); + + // Writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); + taosArrayDestroy(pCommitter->dWriter.aBlockL); tMapDataClear(&pCommitter->dWriter.mBlock); tBlockDataClear(&pCommitter->dWriter.bData, 1); + tBlockDataClear(&pCommitter->dWriter.bDatal, 1); tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 135ee23d44..f9f2c0df6b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -21,6 +21,7 @@ int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) { n += tPutI64v(p ? p + n : p, pHeadFile->commitID); n += tPutI64v(p ? p + n : p, pHeadFile->size); n += tPutI64v(p ? p + n : p, pHeadFile->offset); + n += tPutI64v(p ? p + n : p, pHeadFile->loffset); return n; } @@ -31,6 +32,7 @@ static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) { n += tGetI64v(p + n, &pHeadFile->commitID); n += tGetI64v(p + n, &pHeadFile->size); n += tGetI64v(p + n, &pHeadFile->offset); + n += tGetI64v(p + n, &pHeadFile->loffset); return n; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 7365ac23b8..f33bb9e455 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -597,6 +597,68 @@ _err: return code; } +int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) { + int32_t code = 0; + int64_t offset = pReader->pSet->pHeadF->loffset; + int64_t size = pReader->pSet->pHeadF->offset - offset; + int64_t n; + uint32_t delimiter; + uint8_t *pBuf = NULL; + SBlockL blockl; + + if (!ppBuf) ppBuf = &pBuf; + + // alloc + code = tRealloc(ppBuf, size); + if (code) goto _err; + + // seek + if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // read + n = taosReadFile(pReader->pHeadFD, *ppBuf, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n < size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // check + if (!taosCheckChecksumWhole(*ppBuf, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // decode + n = 0; + n = tGetU32(*ppBuf + n, &delimiter); + ASSERT(delimiter == TSDB_FILE_DLMT); + + taosArrayClear(aBlockL); + while (n < size - sizeof(TSCKSUM)) { + n += tGetBlockL(*ppBuf + n, &blockl); + + if (taosArrayPush(aBlockL, &blockl) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + + ASSERT(n + sizeof(TSCKSUM) == size); + + tFree(pBuf); + return code; + +_err: + tsdbError("vgId:%d read blockl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock, uint8_t **ppBuf) { int32_t code = 0; int64_t offset = pBlockIdx->offset; @@ -1593,6 +1655,53 @@ _err: return code; } +int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) { + int32_t code = 0; + SHeadFile *pHeadFile = &pWriter->fHead; + uint8_t *pBuf = NULL; + int64_t size; + int64_t n; + + // size + size = sizeof(uint32_t); + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { + size += tPutBlockL(NULL, taosArrayGet(aBlockL, iBlockL)); + } + size += sizeof(TSCKSUM); + + // alloc + if (!ppBuf) ppBuf = &pBuf; + code = tRealloc(ppBuf, size); + if (code) goto _err; + + // encode + n = 0; + n += tPutU32(*ppBuf + n, TSDB_FILE_DLMT); + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { + n += tPutBlockL(*ppBuf + n, taosArrayGet(aBlockL, iBlockL)); + } + taosCalcChecksumAppend(0, *ppBuf, size); + + ASSERT(n + sizeof(TSCKSUM) == size); + + // write + n = taosWriteFile(pWriter->pHeadFD, *ppBuf, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // update + pHeadFile->loffset = pHeadFile->size; + pHeadFile->size += size; + + return code; + +_err: + tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + return code; +} + static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 6bb2b8c253..750a5dd164 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -911,14 +911,14 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 if (pSet) { wSet.diskId = pSet->diskId; wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; + fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .loffset = 0}; fData = *pSet->pDataF; fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; fSma = *pSet->pSmaF; } else { wSet.diskId = (SDiskID){.level = 0, .id = 0}; wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; + fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .loffset = 0}; fData = (SDataFile){.commitID = pWriter->commitID, .size = 0}; fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d612e9bb10..4e0d1b2402 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -295,6 +295,56 @@ bool tBlockHasSma(SBlock *pBlock) { return pBlock->aSubBlock[0].nSma > 0; } +// SBlockL ====================================================== +int32_t tPutBlockL(uint8_t *p, void *ph) { + int32_t n = 0; + SBlockL *pBlockL = (SBlockL *)ph; + + n += tPutI64(p ? p + n : p, pBlockL->suid); + n += tPutI64(p ? p + n : p, pBlockL->minKey.uid); + n += tPutI64v(p ? p + n : p, pBlockL->minKey.version); + n += tPutI64(p ? p + n : p, pBlockL->minKey.ts); + n += tPutI64(p ? p + n : p, pBlockL->maxKey.uid); + n += tPutI64v(p ? p + n : p, pBlockL->maxKey.version); + n += tPutI64(p ? p + n : p, pBlockL->maxKey.ts); + n += tPutI64v(p ? p + n : p, pBlockL->minVer); + n += tPutI64v(p ? p + n : p, pBlockL->maxVer); + n += tPutI32v(p ? p + n : p, pBlockL->nRow); + n += tPutI8(p ? p + n : p, pBlockL->cmprAlg); + n += tPutI64v(p ? p + n : p, pBlockL->offset); + n += tPutI32v(p ? p + n : p, pBlockL->szBlock); + n += tPutI32v(p ? p + n : p, pBlockL->szBlockCol); + n += tPutI32v(p ? p + n : p, pBlockL->szUid); + n += tPutI32v(p ? p + n : p, pBlockL->szVer); + n += tPutI32v(p ? p + n : p, pBlockL->szTSKEY); + + return n; +} + +int32_t tGetBlockL(uint8_t *p, void *ph) { + int32_t n = 0; + SBlockL *pBlockL = (SBlockL *)ph; + + n += tGetI64(p + n, &pBlockL->suid); + n += tGetI64(p + n, &pBlockL->minKey.uid); + n += tGetI64v(p + n, &pBlockL->minKey.version); + n += tGetI64(p + n, &pBlockL->minKey.ts); + n += tGetI64(p + n, &pBlockL->maxKey.uid); + n += tGetI64v(p + n, &pBlockL->maxKey.version); + n += tGetI64(p + n, &pBlockL->maxKey.ts); + n += tGetI64v(p + n, &pBlockL->minVer); + n += tGetI64v(p + n, &pBlockL->maxVer); + n += tGetI32v(p + n, &pBlockL->nRow); + n += tGetI8(p + n, &pBlockL->cmprAlg); + n += tGetI64v(p + n, &pBlockL->offset); + n += tGetI32v(p + n, &pBlockL->szBlock); + n += tGetI32v(p + n, &pBlockL->szBlockCol); + n += tGetI32v(p + n, &pBlockL->szUid); + n += tGetI32v(p + n, &pBlockL->szVer); + n += tGetI32v(p + n, &pBlockL->szTSKEY); + + return n; +} // SBlockCol ====================================================== int32_t tPutBlockCol(uint8_t *p, void *ph) { From 2d0882edba25b2e87491a0be4fe2f21df3536382 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 2 Aug 2022 09:06:18 +0000 Subject: [PATCH 002/194] more last file refact --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 226 +++++++++++++++++------ 1 file changed, 165 insertions(+), 61 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3900adf205..2f57992280 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -20,6 +20,12 @@ typedef struct { STSchema *pTSchema; } SSkmInfo; +typedef struct { + int64_t suid; + int64_t uid; + TSDBROW row; +} SRowInfo; + typedef struct { STsdb *pTsdb; /* commit data */ @@ -38,11 +44,20 @@ typedef struct { // commit file data struct { SDataFReader *pReader; - SArray *aBlockIdx; // SArray - SArray *aBlockL; // SArray - SMapData mBlock; // SMapData, read from reader - SBlockData bData; - SBlockData bDatal; + // data + SArray *aBlockIdx; // SArray + int32_t iBlockIdx; + SBlockIdx *pBlockIdx; + SMapData mBlock; // SMapData + SBlockData bData; + // last + SArray *aBlockL; // SArray + int32_t iBlockL; + SBlockL *pBlockL; + SBlockData bDatal; + int32_t iRow; + SRowInfo *pRow; + SRowInfo row; } dReader; struct { SDataFWriter *pWriter; @@ -290,20 +305,46 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet); if (code) goto _err; + // data code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx, NULL); if (code) goto _err; + pCommitter->dReader.iBlockIdx = 0; + if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { + pCommitter->dReader.pBlockIdx = + (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); + + code = + tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); + if (code) goto _err; + } else { + pCommitter->dReader.pBlockIdx = NULL; + } + + // last code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL, NULL); if (code) goto _err; + pCommitter->dReader.iBlockL = 0; + if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { + pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); + + // TODO: code = tsdbReadBlockData(pCommitter->dReader.pReader, NULL, pBlockL, &pCommitter->dReader.bDatal, NULL, + // NULL); + if (code) goto _err; + + pCommitter->dReader.iRow = 0; + pCommitter->dReader.pRow = &pCommitter->dReader.row; + pCommitter->dReader.pRow->suid = pCommitter->dReader.pBlockL->suid; + pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; + pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + } else { + pCommitter->dReader.pRow = NULL; + } } else { - pCommitter->dReader.pReader = NULL; - taosArrayClear(pCommitter->dReader.aBlockIdx); - taosArrayClear(pCommitter->dReader.aBlockL); + pCommitter->dReader.pBlockIdx = NULL; + pCommitter->dReader.pRow = NULL; } - tMapDataReset(&pCommitter->dReader.mBlock); - tBlockDataReset(&pCommitter->dReader.bData); - tBlockDataReset(&pCommitter->dReader.bDatal); // Writer SHeadFile fHead; @@ -707,7 +748,7 @@ _err: return code; } -static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData, SBlockIdx *pBlockIdx) { +static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { int32_t code = 0; STbDataIter iter = {0}; STbDataIter *pIter = &iter; @@ -716,6 +757,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData, SBl int32_t nBlock; int64_t suid; int64_t uid; + SBlockIdx *pBlockIdx = NULL; if (pTbData) { tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, pIter); @@ -908,6 +950,95 @@ _err: return code; } +static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { + int32_t code = 0; + + // data + while (true) { + if (pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) >= 0) break; + + SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; + code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, NULL, &blockIdx); + if (code) goto _err; + + if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + pCommitter->dReader.iBlockIdx++; + if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { + pCommitter->dReader.pBlockIdx = + (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); + + code = + tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); + if (code) goto _err; + } else { + pCommitter->dReader.pBlockIdx = NULL; + } + } + + // last + SBlockL blockL; + while (true) { + if (pCommitter->dReader.pRow == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRow, &toTable) >= 0) break; + + // check if same suid + if (0) { + goto _write_block_data; + } + + // append + code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pCommitter->dReader.pRow->row, NULL); + if (code) goto _err; + + // next + pCommitter->dReader.iRow++; + if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { + pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; + pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + } else { + pCommitter->dReader.iBlockL++; + if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { + pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); + + // TODO: code = tsdbReadBlockData(pCommitter->dReader.pReader, NULL, pBlockL, &pCommitter->dReader.bDatal, NULL, + // NULL); + if (code) goto _err; + + pCommitter->dReader.iRow = 0; + pCommitter->dReader.pRow->suid = pCommitter->dReader.pBlockL->suid; + pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; + pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + } else { + pCommitter->dReader.pRow = NULL; + } + } + + // write + if (pCommitter->dWriter.bDatal.nRow < pCommitter->maxRow) continue; + + _write_block_data: + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, NULL, NULL, NULL, NULL, + pCommitter->cmprAlg); // todo + if (code) goto _err; + + if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + tBlockDataClearData(&pCommitter->dWriter.bDatal); + } + + return code; + +_err: + tsdbError("vgId:%d tsdb move commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} + static int32_t tsdbCommitFileData(SCommitter *pCommitter) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; @@ -918,59 +1049,32 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { if (code) goto _err; // commit file data impl - int32_t iTbData = 0; - int32_t nTbData = taosArrayGetSize(pMemTable->aTbData); - int32_t iBlockIdx = 0; - int32_t nBlockIdx = taosArrayGetSize(pCommitter->dReader.aBlockIdx); - STbData *pTbData; - SBlockIdx *pBlockIdx; + for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pMemTable->aTbData); iTbData++) { + STbData *pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData); - ASSERT(nTbData > 0); + // move commit until current (suid, uid) + code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = pTbData->suid, .uid = pTbData->uid}); + if (code) goto _err; - pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData); - pBlockIdx = (iBlockIdx < nBlockIdx) ? (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, iBlockIdx) : NULL; - while (pTbData || pBlockIdx) { - if (pTbData && pBlockIdx) { - int32_t c = tTABLEIDCmprFn(pTbData, pBlockIdx); + // commit current table data commit + code = tsdbCommitTableData(pCommitter, pTbData); + if (code) goto _err; + } - if (c == 0) { - goto _commit_table_mem_and_disk; - } else if (c < 0) { - goto _commit_table_mem_data; - } else { - goto _commit_table_disk_data; - } - } else if (pBlockIdx) { - goto _commit_table_disk_data; - } else { - goto _commit_table_mem_data; + code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}); + if (code) goto _err; + + if (pCommitter->dWriter.bDatal.nRow > 0) { + SBlockL blockL; + + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, NULL, NULL, NULL, NULL, + pCommitter->cmprAlg); + if (code) goto _err; + + if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } - - _commit_table_mem_data: - code = tsdbCommitTableData(pCommitter, pTbData, NULL); - if (code) goto _err; - - iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData) : NULL; - continue; - - _commit_table_disk_data: - code = tsdbCommitTableData(pCommitter, NULL, pBlockIdx); - if (code) goto _err; - - iBlockIdx++; - pBlockIdx = (iBlockIdx < nBlockIdx) ? (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, iBlockIdx) : NULL; - continue; - - _commit_table_mem_and_disk: - code = tsdbCommitTableData(pCommitter, pTbData, pBlockIdx); - if (code) goto _err; - - iBlockIdx++; - pBlockIdx = (iBlockIdx < nBlockIdx) ? (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, iBlockIdx) : NULL; - iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData) : NULL; - continue; } // commit file data end From f24317fbfff1fefc930d91db5a222e31f2000a76 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 3 Aug 2022 01:45:56 +0000 Subject: [PATCH 003/194] more last file refact --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 211 ++++++++++++++++++----- 1 file changed, 171 insertions(+), 40 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 2f57992280..704aa0aab0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -569,7 +569,8 @@ _err: } static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey, int8_t toDataOnly) { - int32_t code = 0; + int32_t code = 0; +#if 0 TSDBROW *pRow; SBlock block; SBlock *pBlock = █ @@ -623,6 +624,7 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter tBlockDataClearData(pBlockData); } +#endif return code; _err: @@ -653,25 +655,25 @@ _err: return code; } -static int32_t tsdbCommitTableDataEnd(SCommitter *pCommitter, int64_t suid, int64_t uid) { - int32_t code = 0; - SBlockIdx blockIdx = {.suid = suid, .uid = uid}; - SBlockIdx *pBlockIdx = &blockIdx; +// static int32_t tsdbCommitTableDataEnd(SCommitter *pCommitter, int64_t suid, int64_t uid) { +// int32_t code = 0; +// SBlockIdx blockIdx = {.suid = suid, .uid = uid}; +// SBlockIdx *pBlockIdx = &blockIdx; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, NULL, pBlockIdx); - if (code) goto _err; +// code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, NULL, pBlockIdx); +// if (code) goto _err; - if (taosArrayPush(pCommitter->dWriter.aBlockIdx, pBlockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } +// if (taosArrayPush(pCommitter->dWriter.aBlockIdx, pBlockIdx) == NULL) { +// code = TSDB_CODE_OUT_OF_MEMORY; +// goto _err; +// } - return code; +// return code; -_err: - tsdbError("vgId:%d, commit table data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} +// _err: +// tsdbError("vgId:%d, commit table data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +// return code; +// } static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) { int32_t nRow = 0; @@ -748,6 +750,145 @@ _err: return code; } +static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { + int32_t code = 0; + + ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0); + ASSERT(pCommitter->dReader.pRow == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRow, pTbData) >= 0); + + // end last if need + if (pTbData->suid == 0 || pTbData->suid != 0 /*todo*/) { + if (pCommitter->dWriter.bDatal.nRow > 0) { + // TODO: code = tsdbCommitBlockDataL(pCommitter); + if (code) goto _err; + } + } + + // merge commit table data + STbDataIter iter = {0}; + STbDataIter *pIter = &iter; + TSDBROW *pRow; + + tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, pIter); + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + + if (pRow == NULL) goto _exit; + + SBlockIdx *pBlockIdx = NULL; + int32_t iBlock = 0; + SBlock block; + SBlock *pBlock = █ + + if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { + pBlockIdx = pCommitter->dReader.pBlockIdx; + } + + if (pBlockIdx && iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + } else { + pBlock = NULL; + } + + tMapDataReset(&pCommitter->dWriter.mBlock); + while (pBlock && pRow) { + int32_t c = tBlockCmprFn(pBlock, &(SBlock){.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)}); + if (c < 0) { // disk + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); + if (code) goto _err; + + // next + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + } else { + pBlock = NULL; + } + } else if (c < 0) { // memory + code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey, 1); + if (code) goto _err; + + // next + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + } else { // merge + int32_t nOvlp = tsdbGetOvlpNRow(pIter, pBlock); + + ASSERT(nOvlp > 0); + + if (pBlock->nRow + nOvlp <= pCommitter->maxRow && pBlock->nSubBlock < TSDB_MAX_SUBBLOCKS) { + code = tsdbMergeAsSubBlock(pCommitter, pIter, pBlock); + if (code) goto _err; + } else { + // code = tsdbMergeTableData(pCommitter, pIter, pBlock, NULL, 1); + if (code) goto _err; + } + + // next + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + } else { + pBlock = NULL; + } + } + } + + while (pBlock) { + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); + if (code) goto _err; + + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + } else { + pBlock = NULL; + } + } + + if (pRow) { + code = + tsdbCommitTableMemData(pCommitter, pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}, 0); + if (code) goto _err; + + pRow = tsdbTbDataIterGet(pIter); + ASSERT(pRow == NULL || TSDBROW_TS(pRow) > pCommitter->maxKey); + } + + // end + if (pCommitter->dWriter.mBlock.nItem > 0) { + SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid}; + code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, NULL, &blockIdx); + if (code) goto _err; + + if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + +_exit: + pRow = tsdbTbDataIterGet(pIter); + if (pRow) { + pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); + } + + return code; + +_err: + tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +#if 0 static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { int32_t code = 0; STbDataIter iter = {0}; @@ -913,6 +1054,7 @@ _err: tsdbError("vgId:%d, tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } +#endif static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; @@ -985,8 +1127,14 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { if (pCommitter->dReader.pRow == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRow, &toTable) >= 0) break; // check if same suid - if (0) { - goto _write_block_data; + if (pCommitter->dReader.pRow->suid == 0) { + if (pCommitter->dReader.pRow->uid != 0 /*todo*/) { + // code = tsdbCommitBlockDataL(pCommitter); + if (code) goto _err; + } + } else if (pCommitter->dReader.pRow->suid != 0 /*todo*/) { + // code = tsdbCommitBlockDataL(pCommitter); + if (code) goto _err; } // append @@ -1017,19 +1165,10 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { } // write - if (pCommitter->dWriter.bDatal.nRow < pCommitter->maxRow) continue; - - _write_block_data: - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, NULL, NULL, NULL, NULL, - pCommitter->cmprAlg); // todo - if (code) goto _err; - - if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) { + // code = tsdbCommitBlockDataL(pCommitter); + if (code) goto _err; } - - tBlockDataClearData(&pCommitter->dWriter.bDatal); } return code; @@ -1056,7 +1195,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = pTbData->suid, .uid = pTbData->uid}); if (code) goto _err; - // commit current table data commit + // commit current table data code = tsdbCommitTableData(pCommitter, pTbData); if (code) goto _err; } @@ -1065,16 +1204,8 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { if (code) goto _err; if (pCommitter->dWriter.bDatal.nRow > 0) { - SBlockL blockL; - - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, NULL, NULL, NULL, NULL, - pCommitter->cmprAlg); + // code = tsdbCommitBlockDataL(pCommitter); if (code) goto _err; - - if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } } // commit file data end From e827d0bb7a5e58643f38bd21b6a28a2a5ff28c90 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 3 Aug 2022 08:50:36 +0000 Subject: [PATCH 004/194] more last file refact --- source/dnode/vnode/src/inc/tsdb.h | 13 + source/dnode/vnode/src/tsdb/tsdbCommit.c | 500 +++++++++--------- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 34 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 + 4 files changed, 289 insertions(+), 260 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 8a6f473dc4..af256c0e19 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -233,6 +233,12 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_ SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); + +/* new */ +int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, + uint8_t **ppBuf2, int8_t cmprAlg); +int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, + uint8_t **ppBuf2, int8_t cmprAlg); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); @@ -244,6 +250,12 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf); + +/* new */ +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2); +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2); // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); @@ -457,6 +469,7 @@ struct SColData { }; struct SBlockData { + int64_t suid; int32_t nRow; int64_t *aUid; int64_t *aVersion; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 704aa0aab0..70892acb0a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -56,8 +56,8 @@ typedef struct { SBlockL *pBlockL; SBlockData bDatal; int32_t iRow; - SRowInfo *pRow; - SRowInfo row; + SRowInfo *pRowInfo; + SRowInfo rowInfo; } dReader; struct { SDataFWriter *pWriter; @@ -290,12 +290,46 @@ _err: return code; } +static int32_t tsdbCommitNextLastRow(SCommitter *pCommitter) { + int32_t code = 0; + + ASSERT(pCommitter->dReader.pReader); + ASSERT(pCommitter->dReader.pRowInfo); + + pCommitter->dReader.iRow++; + if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { + pCommitter->dReader.pRowInfo->uid = pCommitter->dReader.bData.aUid[pCommitter->dReader.iRow]; + pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + } else { + pCommitter->dReader.iBlockL++; + if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { + pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); + code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, &pCommitter->dReader.bDatal, + NULL, NULL); + if (code) goto _exit; + + pCommitter->dReader.iRow = 0; + pCommitter->dReader.pRowInfo->suid = pCommitter->dReader.pBlockL->suid; + pCommitter->dReader.pRowInfo->uid = pCommitter->dReader.bData.aUid[pCommitter->dReader.iRow]; + pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + } else { + pCommitter->dReader.pRowInfo = NULL; + } + } + +_exit: + return code; +} + static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; SDFileSet *pRSet = NULL; // memory + pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision); + tsdbFidKeyRange(pCommitter->commitFid, pCommitter->minutes, pCommitter->precision, &pCommitter->minKey, + &pCommitter->maxKey); pCommitter->nextKey = TSKEY_MAX; // Reader @@ -325,25 +359,15 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL, NULL); if (code) goto _err; - pCommitter->dReader.iBlockL = 0; - if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - - // TODO: code = tsdbReadBlockData(pCommitter->dReader.pReader, NULL, pBlockL, &pCommitter->dReader.bDatal, NULL, - // NULL); - if (code) goto _err; - - pCommitter->dReader.iRow = 0; - pCommitter->dReader.pRow = &pCommitter->dReader.row; - pCommitter->dReader.pRow->suid = pCommitter->dReader.pBlockL->suid; - pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; - pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); - } else { - pCommitter->dReader.pRow = NULL; - } + pCommitter->dReader.iBlockL = -1; + pCommitter->dReader.bDatal.nRow = 0; + pCommitter->dReader.iRow = -1; + pCommitter->dReader.pRowInfo = &pCommitter->dReader.rowInfo; + code = tsdbCommitNextLastRow(pCommitter); + if (code) goto _err; } else { pCommitter->dReader.pBlockIdx = NULL; - pCommitter->dReader.pRow = NULL; + pCommitter->dReader.pRowInfo = NULL; } // Writer @@ -463,6 +487,46 @@ _err: return code; } +static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) { + int32_t code = 0; + SBlock block; + + ASSERT(pCommitter->dWriter.bData.nRow > 0); + + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bData, &block, NULL, NULL, + pCommitter->cmprAlg); + if (code) goto _exit; + + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); + if (code) goto _exit; + + tBlockDataClearData(&pCommitter->dWriter.bData); + +_exit: + return code; +} + +static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { + int32_t code = 0; + SBlockL blockL; + + ASSERT(pCommitter->dWriter.bDatal.nRow > 0); + + code = tsdbWriteLastBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, &blockL, NULL, NULL, + pCommitter->cmprAlg); + if (code) goto _exit; + + if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + tBlockDataClearData(&pCommitter->dWriter.bDatal); + +_exit: + return code; +} + static int32_t tsdbMergeTableData(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlockMerge, TSDBKEY toKey, int8_t toDataOnly) { int32_t code = 0; @@ -675,22 +739,16 @@ _err: // return code; // } -static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) { - int32_t nRow = 0; - TSDBROW *pRow; - TSDBKEY key; - int32_t c = 0; +static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) { + int32_t nRow = 0; + STbDataIter iter = *pIter; - - iter.pRow = NULL; while (true) { - pRow = tsdbTbDataIterGet(&iter); - + TSDBROW *pRow = tsdbTbDataIterGet(&iter); if (pRow == NULL) break; - key = TSDBROW_KEY(pRow); - c = tBlockCmprFn(&(SBlock){.maxKey = key, .minKey = key}, pBlock); - if (c == 0) { + int32_t c = tsdbKeyCmprFn(&TSDBROW_KEY(pRow), &key); + if (c < 0) { nRow++; tsdbTbDataIterNext(&iter); } else if (c > 0) { @@ -750,11 +808,106 @@ _err: return code; } +static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter, int32_t *nRow) { + int32_t code = 0; + STbData *pTbData = pIter->pTbData; + TSDBROW *pRow = tsdbTbDataIterGet(pIter); + SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; + + while (pRow && pRowInfo) { + int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); + if (c < 0) { + code = tBlockDataAppendRow(&pCommitter->dWriter.bData, pRow, NULL); + if (code) goto _err; + + tsdbTbDataIterNext(pIter); + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + } else if (c > 0) { + code = tBlockDataAppendRow(&pCommitter->dWriter.bData, &pRowInfo->row, NULL); + if (code) goto _err; + + pCommitter->dReader.iRow++; + if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { + // todo + } else { + pCommitter->dReader.iBlockL++; + if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { + // todo + } else { + // todo + } + } + + pRowInfo = pCommitter->dReader.pRowInfo; + if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + pRowInfo = NULL; + } + } else { + ASSERT(0); + } + + (*nRow)--; + ASSERT(*nRow >= 0); + if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; + } + + while (pRow) { + code = tBlockDataAppendRow(&pCommitter->dWriter.bData, pRow, NULL); + if (code) goto _err; + + tsdbTbDataIterNext(pIter); + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + + (*nRow)--; + ASSERT(*nRow >= 0); + if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; + } + + while (pRowInfo) { + code = tBlockDataAppendRow(&pCommitter->dWriter.bData, &pRowInfo->row, NULL); + if (code) goto _err; + + pCommitter->dReader.iRow++; + if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { + // todo + } else { + pCommitter->dReader.iBlockL++; + if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { + // todo + } else { + // todo + } + } + + pRowInfo = pCommitter->dReader.pRowInfo; + if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + pRowInfo = NULL; + } + + (*nRow)--; + ASSERT(*nRow >= 0); + if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; + } + + SBlock block; +_write_block_data: + return code; + +_err: + return code; +} + static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { int32_t code = 0; ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0); - ASSERT(pCommitter->dReader.pRow == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRow, pTbData) >= 0); + ASSERT(pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, pTbData) >= 0); // end last if need if (pTbData->suid == 0 || pTbData->suid != 0 /*todo*/) { @@ -816,7 +969,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { pRow = NULL; } } else { // merge - int32_t nOvlp = tsdbGetOvlpNRow(pIter, pBlock); + int32_t nOvlp = tsdbGetNumOfRowsLessThan(pIter, pBlock->maxKey); ASSERT(nOvlp > 0); @@ -854,13 +1007,20 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { } } - if (pRow) { - code = - tsdbCommitTableMemData(pCommitter, pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}, 0); - if (code) goto _err; + // merge with last + int32_t nRowLeft = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); + if (pCommitter->dReader.pRowInfo) { + for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { + int64_t uid = pCommitter->dReader.bDatal.aUid[iRow]; + if (uid == pTbData->uid) { + nRowLeft++; + } + } + } - pRow = tsdbTbDataIterGet(pIter); - ASSERT(pRow == NULL || TSDBROW_TS(pRow) > pCommitter->maxKey); + while (nRowLeft) { + code = tsdbMergeCommitLast(pCommitter, pIter, &nRowLeft); + if (code) goto _err; } // end @@ -888,174 +1048,6 @@ _err: return code; } -#if 0 -static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { - int32_t code = 0; - STbDataIter iter = {0}; - STbDataIter *pIter = &iter; - TSDBROW *pRow; - int32_t iBlock; - int32_t nBlock; - int64_t suid; - int64_t uid; - SBlockIdx *pBlockIdx = NULL; - - if (pTbData) { - tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) pRow = NULL; - - suid = pTbData->suid; - uid = pTbData->uid; - } else { - pIter = NULL; - pRow = NULL; - } - - if (pBlockIdx) { - code = tsdbReadBlock(pCommitter->dReader.pReader, pBlockIdx, &pCommitter->dReader.mBlock, NULL); - if (code) goto _err; - - nBlock = pCommitter->dReader.mBlock.nItem; - ASSERT(nBlock > 0); - - suid = pBlockIdx->suid; - uid = pBlockIdx->uid; - } else { - nBlock = 0; - } - - if (pRow == NULL && nBlock == 0) goto _exit; - - // start =========== - tMapDataReset(&pCommitter->dWriter.mBlock); - SBlock block; - SBlock *pBlock = █ - - iBlock = 0; - if (iBlock < nBlock) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - - if (pRow) { - code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid, pTbData->maxSkmVer); - if (code) goto _err; - } - - // merge =========== - while (true) { - if (pRow == NULL && pBlock == NULL) break; - - if (pRow && pBlock) { - if (pBlock->last) { - code = tsdbMergeTableData(pCommitter, pIter, pBlock, - (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}, 0); - if (code) goto _err; - - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) pRow = NULL; - iBlock++; - if (iBlock < nBlock) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - - ASSERT(pRow == NULL && pBlock == NULL); - } else { - int32_t c = tBlockCmprFn(&(SBlock){.maxKey = TSDBROW_KEY(pRow), .minKey = TSDBROW_KEY(pRow)}, pBlock); - if (c > 0) { - // only disk data - code = tsdbCommitTableDiskData(pCommitter, pBlock, pBlockIdx); - if (code) goto _err; - - iBlock++; - if (iBlock < nBlock) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } else if (c < 0) { - // only memory data - code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey, 1); - if (code) goto _err; - - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) pRow = NULL; - } else { - // merge memory and disk - int32_t nOvlp = tsdbGetOvlpNRow(pIter, pBlock); - ASSERT(nOvlp); - if (pBlock->nRow + nOvlp <= pCommitter->maxRow && pBlock->nSubBlock < TSDB_MAX_SUBBLOCKS) { - code = tsdbMergeAsSubBlock(pCommitter, pIter, pBlock); - if (code) goto _err; - } else { - TSDBKEY toKey = {.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}; - int8_t toDataOnly = 0; - - if (iBlock < nBlock - 1) { - toDataOnly = 1; - - SBlock nextBlock = {0}; - tBlockReset(&nextBlock); - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock + 1, &nextBlock, tGetBlock); - toKey = nextBlock.minKey; - } - - code = tsdbMergeTableData(pCommitter, pIter, pBlock, toKey, toDataOnly); - if (code) goto _err; - } - - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) pRow = NULL; - iBlock++; - if (iBlock < nBlock) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } - } - } else if (pBlock) { - code = tsdbCommitTableDiskData(pCommitter, pBlock, pBlockIdx); - if (code) goto _err; - - iBlock++; - if (iBlock < nBlock) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } else { - code = - tsdbCommitTableMemData(pCommitter, pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}, 0); - if (code) goto _err; - - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) pRow = NULL; - ASSERT(pRow == NULL); - } - } - - // end ===================== - code = tsdbCommitTableDataEnd(pCommitter, suid, uid); - if (code) goto _err; - -_exit: - if (pIter) { - pRow = tsdbTbDataIterGet(pIter); - if (pRow) pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); - } - return code; - -_err: - tsdbError("vgId:%d, tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} -#endif - static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; @@ -1122,52 +1114,53 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { } // last - SBlockL blockL; while (true) { - if (pCommitter->dReader.pRow == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRow, &toTable) >= 0) break; + if (pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, &toTable) >= 0) break; - // check if same suid - if (pCommitter->dReader.pRow->suid == 0) { - if (pCommitter->dReader.pRow->uid != 0 /*todo*/) { - // code = tsdbCommitBlockDataL(pCommitter); + // commit if not same schema + if (pCommitter->dWriter.bDatal.nRow > 0) { + if (pCommitter->dWriter.bDatal.suid != pCommitter->dReader.pRowInfo->suid || + pCommitter->dWriter.bDatal.suid == 0) { + code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; } - } else if (pCommitter->dReader.pRow->suid != 0 /*todo*/) { - // code = tsdbCommitBlockDataL(pCommitter); - if (code) goto _err; - } - - // append - code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pCommitter->dReader.pRow->row, NULL); - if (code) goto _err; - - // next - pCommitter->dReader.iRow++; - if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { - pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; - pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); - } else { - pCommitter->dReader.iBlockL++; - if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - - // TODO: code = tsdbReadBlockData(pCommitter->dReader.pReader, NULL, pBlockL, &pCommitter->dReader.bDatal, NULL, - // NULL); - if (code) goto _err; - - pCommitter->dReader.iRow = 0; - pCommitter->dReader.pRow->suid = pCommitter->dReader.pBlockL->suid; - pCommitter->dReader.pRow->uid = pCommitter->dReader.bDatal.aUid[pCommitter->dReader.iRow]; - pCommitter->dReader.pRow->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); - } else { - pCommitter->dReader.pRow = NULL; - } } - // write - if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) { - // code = tsdbCommitBlockDataL(pCommitter); + if (pCommitter->dWriter.bDatal.nRow == 0) { + code = tsdbCommitterUpdateTableSchema(pCommitter, pCommitter->dReader.pRowInfo->suid, + pCommitter->dReader.pRowInfo->suid, 1 /*TODO*/); if (code) goto _err; + + pCommitter->dWriter.bDatal.suid = pCommitter->dReader.pRowInfo->suid; + code = tBlockDataSetSchema(&pCommitter->dWriter.bDatal, pCommitter->skmTable.pTSchema); + if (code) goto _err; + } + + // check if it can make sure that one table data in one block + int64_t uid = pCommitter->dReader.pRowInfo->uid; + int32_t nRow = 0; + for (int32_t iRow = pCommitter->dReader.iRow; + (iRow < pCommitter->dReader.bDatal.nRow) && (pCommitter->dReader.bDatal.aUid[iRow] == uid); iRow++) { + nRow++; + } + + ASSERT(nRow > 0 && nRow < pCommitter->minRow); + + if (pCommitter->dWriter.bDatal.nRow + nRow > pCommitter->maxRow) { + ASSERT(pCommitter->dWriter.bDatal.nRow > 0); + + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } + + while (nRow > 0) { + code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pCommitter->dReader.pRowInfo->row, NULL); + if (code) goto _err; + + code = tsdbCommitNextLastRow(pCommitter); + if (code) goto _err; + + nRow--; } } @@ -1192,7 +1185,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { STbData *pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData); // move commit until current (suid, uid) - code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = pTbData->suid, .uid = pTbData->uid}); + code = tsdbMoveCommitData(pCommitter, *(TABLEID *)pTbData); if (code) goto _err; // commit current table data @@ -1204,7 +1197,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { if (code) goto _err; if (pCommitter->dWriter.bDatal.nRow > 0) { - // code = tsdbCommitBlockDataL(pCommitter); + code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; } @@ -1261,15 +1254,15 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { goto _exit; } + code = tBlockDataInit(&pCommitter->dReader.bData); + if (code) goto _exit; + pCommitter->dReader.aBlockL = taosArrayInit(0, sizeof(SBlockL)); if (pCommitter->dReader.aBlockL == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - code = tBlockDataInit(&pCommitter->dReader.bData); - if (code) goto _exit; - code = tBlockDataInit(&pCommitter->dReader.bDatal); if (code) goto _exit; @@ -1299,9 +1292,9 @@ _exit: static void tsdbCommitDataEnd(SCommitter *pCommitter) { // Reader taosArrayDestroy(pCommitter->dReader.aBlockIdx); - taosArrayDestroy(pCommitter->dReader.aBlockL); tMapDataClear(&pCommitter->dReader.mBlock); tBlockDataClear(&pCommitter->dReader.bData, 1); + taosArrayDestroy(pCommitter->dReader.aBlockL); tBlockDataClear(&pCommitter->dReader.bDatal, 1); // Writer @@ -1329,9 +1322,6 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { // impl ==================== pCommitter->nextKey = pMemTable->minKey; while (pCommitter->nextKey < TSKEY_MAX) { - pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision); - tsdbFidKeyRange(pCommitter->commitFid, pCommitter->minutes, pCommitter->precision, &pCommitter->minKey, - &pCommitter->maxKey); code = tsdbCommitFileData(pCommitter); if (code) goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 22615b9508..79929c44a0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -543,9 +543,13 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB uint32_t delimiter; SBlockIdx blockIdx; - if (!ppBuf) ppBuf = &pBuf; + taosArrayClear(aBlockIdx); + if (size == 0) { + goto _exit; + } // alloc + if (!ppBuf) ppBuf = &pBuf; code = tRealloc(ppBuf, size); if (code) goto _err; @@ -576,7 +580,6 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB n = tGetU32(*ppBuf + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); - taosArrayClear(aBlockIdx); while (n < size - sizeof(TSCKSUM)) { n += tGetBlockIdx(*ppBuf + n, &blockIdx); @@ -588,6 +591,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB ASSERT(n + sizeof(TSCKSUM) == size); +_exit: tFree(pBuf); return code; @@ -606,9 +610,13 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) uint8_t *pBuf = NULL; SBlockL blockl; - if (!ppBuf) ppBuf = &pBuf; + taosArrayClear(aBlockL); + if (size == 0) { + goto _exit; + } // alloc + if (!ppBuf) ppBuf = &pBuf; code = tRealloc(ppBuf, size); if (code) goto _err; @@ -639,7 +647,6 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) n = tGetU32(*ppBuf + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); - taosArrayClear(aBlockL); while (n < size - sizeof(TSCKSUM)) { n += tGetBlockL(*ppBuf + n, &blockl); @@ -651,11 +658,13 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) ASSERT(n + sizeof(TSCKSUM) == size); +_exit: tFree(pBuf); return code; _err: tsdbError("vgId:%d read blockl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tFree(pBuf); return code; } @@ -1562,7 +1571,11 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **pp int64_t size; int64_t n; - if (!ppBuf) ppBuf = &pBuf; + // check + if (taosArrayGetSize(aBlockIdx) == 0) { + pHeadFile->offset = pHeadFile->size; + goto _exit; + } // prepare size = tPutU32(NULL, TSDB_FILE_DLMT); @@ -1572,6 +1585,7 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **pp size += sizeof(TSCKSUM); // alloc + if (!ppBuf) ppBuf = &pBuf; code = tRealloc(ppBuf, size); if (code) goto _err; @@ -1596,6 +1610,7 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **pp pHeadFile->offset = pHeadFile->size; pHeadFile->size += size; +_exit: tFree(pBuf); return code; @@ -1662,6 +1677,12 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) int64_t size; int64_t n; + // check + if (taosArrayGetSize(aBlockL) == 0) { + pHeadFile->loffset = pHeadFile->size; + goto _exit; + } + // size size = sizeof(uint32_t); for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { @@ -1695,10 +1716,13 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) pHeadFile->loffset = pHeadFile->size; pHeadFile->size += size; +_exit: + tFree(pBuf); return code; _err: tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tFree(pBuf); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 4e0d1b2402..205d8b29dc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1122,6 +1122,7 @@ int32_t tBlockDataInit(SBlockData *pBlockData) { int32_t code = 0; pBlockData->nRow = 0; + pBlockData->aUid = NULL; pBlockData->aVersion = NULL; pBlockData->aTSKEY = NULL; pBlockData->aIdx = taosArrayInit(0, sizeof(int32_t)); @@ -1146,6 +1147,7 @@ void tBlockDataReset(SBlockData *pBlockData) { } void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear) { + tFree((uint8_t *)pBlockData->aUid); tFree((uint8_t *)pBlockData->aVersion); tFree((uint8_t *)pBlockData->aTSKEY); taosArrayDestroy(pBlockData->aIdx); From 7075244183a133bcac4973341f014f5be394225c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 3 Aug 2022 12:16:57 +0000 Subject: [PATCH 005/194] more last file refact --- source/dnode/vnode/src/inc/tsdb.h | 3 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 485 +++++++++--------- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 +- 3 files changed, 240 insertions(+), 252 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index af256c0e19..4c31b3e07d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -469,7 +469,8 @@ struct SColData { }; struct SBlockData { - int64_t suid; + int64_t suid; // 0 means normal table data block + int64_t uid; // 0 means block data in .last file, others in .data file int32_t nRow; int64_t *aUid; int64_t *aVersion; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 70892acb0a..63aa5f3ec8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -290,7 +290,7 @@ _err: return code; } -static int32_t tsdbCommitNextLastRow(SCommitter *pCommitter) { +static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { int32_t code = 0; ASSERT(pCommitter->dReader.pReader); @@ -363,7 +363,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { pCommitter->dReader.bDatal.nRow = 0; pCommitter->dReader.iRow = -1; pCommitter->dReader.pRowInfo = &pCommitter->dReader.rowInfo; - code = tsdbCommitNextLastRow(pCommitter); + code = tsdbCommitterNextLastRow(pCommitter); if (code) goto _err; } else { pCommitter->dReader.pBlockIdx = NULL; @@ -527,128 +527,94 @@ _exit: return code; } -static int32_t tsdbMergeTableData(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlockMerge, TSDBKEY toKey, - int8_t toDataOnly) { +static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { int32_t code = 0; - SBlockIdx *pBlockIdx = &(SBlockIdx){.suid = pIter->pTbData->suid, .uid = pIter->pTbData->uid}; - SBlockData *pBlockDataMerge = &pCommitter->dReader.bData; - SBlockData *pBlockData = &pCommitter->dWriter.bData; - SBlock block; - SBlock *pBlock = █ - TSDBROW *pRow1; - TSDBROW row2; - TSDBROW *pRow2 = &row2; + STbData *pTbData = pIter->pTbData; + SBlockData *pBlockDataR = &pCommitter->dReader.bData; + SBlockData *pBlockDataW = &pCommitter->dWriter.bData; - // read SBlockData - code = tsdbReadBlockData(pCommitter->dReader.pReader, pBlockIdx, pBlockMerge, pBlockDataMerge, NULL, NULL); + code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR, NULL, NULL); if (code) goto _err; - code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema); - if (code) goto _err; + tBlockDataClearData(pBlockDataW); + int32_t iRow = 0; + TSDBROW row; + TSDBROW *pRow1 = tsdbTbDataIterGet(pIter); + TSDBROW *pRow2 = &row; + *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); + while (pRow1 && pRow2) { + int32_t c = tsdbRowCmprFn(pRow1, pRow2); - // loop to merge - pRow1 = tsdbTbDataIterGet(pIter); - *pRow2 = tsdbRowFromBlockData(pBlockDataMerge, 0); - ASSERT(pRow1 && tsdbKeyCmprFn(&TSDBROW_KEY(pRow1), &toKey) < 0); - ASSERT(tsdbKeyCmprFn(&TSDBROW_KEY(pRow2), &toKey) < 0); - code = tsdbCommitterUpdateRowSchema(pCommitter, pBlockIdx->suid, pBlockIdx->uid, TSDBROW_SVERSION(pRow1)); - if (code) goto _err; + if (c < 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow1)); + if (code) goto _err; - tBlockReset(pBlock); - tBlockDataClearData(pBlockData); - while (true) { - if (pRow1 == NULL && pRow2 == NULL) { - if (pBlockData->nRow == 0) { - break; + code = tBlockDataAppendRow(pBlockDataW, pRow1, pCommitter->skmRow.pTSchema); + if (code) goto _err; + + // next + tsdbTbDataIterNext(pIter); + pRow1 = tsdbTbDataIterGet(pIter); + } else if (c > 0) { + code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL); + if (code) goto _err; + + iRow++; + if (iRow < pBlockDataR->nRow) { + *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); } else { - goto _write_block; + pRow2 = NULL; } - } - - if (pRow1 && pRow2) { - int32_t c = tsdbRowCmprFn(pRow1, pRow2); - if (c < 0) { - goto _append_mem_row; - } else if (c > 0) { - goto _append_block_row; - } else { - ASSERT(0); - } - } else if (pRow1) { - goto _append_mem_row; } else { - goto _append_block_row; + ASSERT(0); } - _append_mem_row: - code = tBlockDataAppendRow(pBlockData, pRow1, pCommitter->skmRow.pTSchema); + // check + if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; + } + } + + while (pRow2) { + code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL); if (code) goto _err; - tsdbTbDataIterNext(pIter); - pRow1 = tsdbTbDataIterGet(pIter); - if (pRow1) { - if (tsdbKeyCmprFn(&TSDBROW_KEY(pRow1), &toKey) < 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, pBlockIdx->suid, pBlockIdx->uid, TSDBROW_SVERSION(pRow1)); - if (code) goto _err; - } else { - pRow1 = NULL; - } - } - - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - goto _write_block; - } else { - continue; - } - - _append_block_row: - code = tBlockDataAppendRow(pBlockData, pRow2, NULL); - if (code) goto _err; - - if (pRow2->iRow + 1 < pBlockDataMerge->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockDataMerge, pRow2->iRow + 1); + iRow++; + if (iRow < pBlockDataR->nRow) { + *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); } else { pRow2 = NULL; } - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - goto _write_block; - } else { - continue; + // check + if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; } + } - _write_block: - code = tsdbCommitBlockData(pCommitter, pBlockData, pBlock, pBlockIdx, toDataOnly); + // check + if (pBlockDataW->nRow > 0) { + code = tsdbCommitDataBlock(pCommitter); if (code) goto _err; - - tBlockReset(pBlock); - tBlockDataClearData(pBlockData); } return code; _err: - tsdbError("vgId:%d, tsdb merge block and mem failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, tsdb merge commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } -static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey, int8_t toDataOnly) { - int32_t code = 0; -#if 0 - TSDBROW *pRow; - SBlock block; - SBlock *pBlock = █ +static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey) { + int32_t code = 0; + STbData *pTbData = pIter->pTbData; SBlockData *pBlockData = &pCommitter->dWriter.bData; - int64_t suid = pIter->pTbData->suid; - int64_t uid = pIter->pTbData->uid; - code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema); - if (code) goto _err; - - tBlockReset(pBlock); tBlockDataClearData(pBlockData); - pRow = tsdbTbDataIterGet(pIter); - ASSERT(pRow && tsdbKeyCmprFn(&TSDBROW_KEY(pRow), &toKey) < 0); + + TSDBROW *pRow = tsdbTbDataIterGet(pIter); while (true) { if (pRow == NULL) { if (pBlockData->nRow > 0) { @@ -659,7 +625,7 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter } // update schema - code = tsdbCommitterUpdateRowSchema(pCommitter, suid, uid, TSDBROW_SVERSION(pRow)); + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); if (code) goto _err; // append @@ -668,27 +634,20 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter tsdbTbDataIterNext(pIter); pRow = tsdbTbDataIterGet(pIter); - // if (pRow && tsdbKeyCmprFn(&TSDBROW_KEY(pRow), &toKey) >= 0) pRow = NULL; - // crash on CI, use the block following if (pRow) { - TSDBKEY tmpKey = TSDBROW_KEY(pRow); - if (tsdbKeyCmprFn(&tmpKey, &toKey) >= 0) { + TSDBKEY rowKey = TSDBROW_KEY(pRow); + if (tsdbKeyCmprFn(&rowKey, &toKey) >= 0) { pRow = NULL; } } - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) goto _write_block; - continue; - - _write_block: - code = tsdbCommitBlockData(pCommitter, pBlockData, pBlock, &(SBlockIdx){.suid = suid, .uid = uid}, toDataOnly); - if (code) goto _err; - - tBlockReset(pBlock); - tBlockDataClearData(pBlockData); + if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + _write_block: + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; + } } -#endif return code; _err: @@ -719,26 +678,6 @@ _err: return code; } -// static int32_t tsdbCommitTableDataEnd(SCommitter *pCommitter, int64_t suid, int64_t uid) { -// int32_t code = 0; -// SBlockIdx blockIdx = {.suid = suid, .uid = uid}; -// SBlockIdx *pBlockIdx = &blockIdx; - -// code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, NULL, pBlockIdx); -// if (code) goto _err; - -// if (taosArrayPush(pCommitter->dWriter.aBlockIdx, pBlockIdx) == NULL) { -// code = TSDB_CODE_OUT_OF_MEMORY; -// goto _err; -// } - -// return code; - -// _err: -// tsdbError("vgId:%d, commit table data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); -// return code; -// } - static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) { int32_t nRow = 0; @@ -763,42 +702,35 @@ static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) { static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { int32_t code = 0; + STbData *pTbData = pIter->pTbData; SBlockData *pBlockData = &pCommitter->dWriter.bData; - SBlockIdx *pBlockIdx = &(SBlockIdx){.suid = pIter->pTbData->suid, .uid = pIter->pTbData->uid}; - SBlock block; - TSDBROW *pRow; - code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema); - if (code) goto _err; - - pRow = tsdbTbDataIterGet(pIter); - code = tsdbCommitterUpdateRowSchema(pCommitter, pBlockIdx->suid, pBlockIdx->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; + tBlockDataClearData(pBlockData); while (true) { - if (pRow == NULL) break; - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); - if (code) goto _err; - - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); + TSDBROW *pRow = tsdbTbDataIterGet(pIter); if (pRow) { - TSDBKEY key = TSDBROW_KEY(pRow); - int32_t c = tBlockCmprFn(&(SBlock){.minKey = key, .maxKey = key}, pBlock); - - if (c == 0) { - code = - tsdbCommitterUpdateRowSchema(pCommitter, pIter->pTbData->suid, pIter->pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; - } else if (c > 0) { + TSDBKEY rowKey = TSDBROW_KEY(pRow); + if (tsdbKeyCmprFn(&rowKey, &pBlock->maxKey) > 0) { pRow = NULL; - } else { - ASSERT(0); } } + + if (pRow == NULL) { + break; + } + + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); + if (code) goto _err; } - block = *pBlock; - code = tsdbCommitBlockData(pCommitter, pBlockData, &block, pBlockIdx, 0); + SBlock block = *pBlock; + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &block, NULL, NULL, pCommitter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); if (code) goto _err; return code; @@ -808,16 +740,101 @@ _err: return code; } -static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter, int32_t *nRow) { - int32_t code = 0; - STbData *pTbData = pIter->pTbData; - TSDBROW *pRow = tsdbTbDataIterGet(pIter); - SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; +static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { + int32_t code = 0; + STbData *pTbData = pIter->pTbData; + int32_t nRow = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); - while (pRow && pRowInfo) { - int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); - if (c < 0) { - code = tBlockDataAppendRow(&pCommitter->dWriter.bData, pRow, NULL); + if (pCommitter->dReader.pRowInfo) { + for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { + if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break; + nRow++; + } + } + + if (nRow == 0) goto _exit; + + SBlockData *pBlockData; + TSDBROW *pRow = tsdbTbDataIterGet(pIter); + SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; + + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + + if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + pRowInfo = NULL; + } + + while (nRow) { + if (nRow < pCommitter->minRow) { // to .last + pBlockData = &pCommitter->dWriter.bDatal; + + // check if same schema + if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + + code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema); + if (code) goto _err; + } + + if (pBlockData->nRow + nRow > pCommitter->maxRow) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } + } else { // to .data + pBlockData = &pCommitter->dWriter.bData; + } + + while (pRow && pRowInfo) { + int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); + if (c < 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); + if (code) goto _err; + + tsdbTbDataIterNext(pIter); + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + } else if (c > 0) { + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL); + if (code) goto _err; + + code = tsdbCommitterNextLastRow(pCommitter); + if (code) goto _err; + + pRowInfo = pCommitter->dReader.pRowInfo; + if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + pRowInfo = NULL; + } + } else { + ASSERT(0); + } + + nRow--; + + if (pBlockData->uid) { // .data block + if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; + + goto _outer_break; + } + } else { + ASSERT(pBlockData->nRow <= pCommitter->maxRow); + } + } + + while (pRow) { + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); if (code) goto _err; tsdbTbDataIterNext(pIter); @@ -825,81 +842,54 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter, i if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { pRow = NULL; } - } else if (c > 0) { - code = tBlockDataAppendRow(&pCommitter->dWriter.bData, &pRowInfo->row, NULL); + + nRow--; + if (pBlockData->uid) { // .data block + if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; + + goto _outer_break; + } + } else { + ASSERT(pBlockData->nRow <= pCommitter->maxRow); + } + } + + while (pRowInfo) { + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL); if (code) goto _err; - pCommitter->dReader.iRow++; - if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { - // todo - } else { - pCommitter->dReader.iBlockL++; - if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - // todo - } else { - // todo - } - } + code = tsdbCommitterNextLastRow(pCommitter); + if (code) goto _err; pRowInfo = pCommitter->dReader.pRowInfo; if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { pRowInfo = NULL; } - } else { - ASSERT(0); - } - (*nRow)--; - ASSERT(*nRow >= 0); - if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; - } + nRow--; + if (pBlockData->uid) { // .data block + if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; - while (pRow) { - code = tBlockDataAppendRow(&pCommitter->dWriter.bData, pRow, NULL); - if (code) goto _err; - - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - - (*nRow)--; - ASSERT(*nRow >= 0); - if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; - } - - while (pRowInfo) { - code = tBlockDataAppendRow(&pCommitter->dWriter.bData, &pRowInfo->row, NULL); - if (code) goto _err; - - pCommitter->dReader.iRow++; - if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { - // todo - } else { - pCommitter->dReader.iBlockL++; - if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - // todo + goto _outer_break; + } } else { - // todo + ASSERT(pBlockData->nRow <= pCommitter->maxRow); } } - pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { - pRowInfo = NULL; - } - - (*nRow)--; - ASSERT(*nRow >= 0); - if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) goto _write_block_data; + _outer_break: + ASSERT(nRow >= 0); } - SBlock block; -_write_block_data: +_exit: return code; _err: + tsdbError("vgId:%d tsdb merge commit last failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } @@ -909,14 +899,6 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0); ASSERT(pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, pTbData) >= 0); - // end last if need - if (pTbData->suid == 0 || pTbData->suid != 0 /*todo*/) { - if (pCommitter->dWriter.bDatal.nRow > 0) { - // TODO: code = tsdbCommitBlockDataL(pCommitter); - if (code) goto _err; - } - } - // merge commit table data STbDataIter iter = {0}; STbDataIter *pIter = &iter; @@ -930,22 +912,22 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { if (pRow == NULL) goto _exit; - SBlockIdx *pBlockIdx = NULL; - int32_t iBlock = 0; - SBlock block; - SBlock *pBlock = █ - - if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { - pBlockIdx = pCommitter->dReader.pBlockIdx; - } - - if (pBlockIdx && iBlock < pCommitter->dReader.mBlock.nItem) { + int32_t iBlock = 0; + SBlock block; + SBlock *pBlock = █ + if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0 && + iBlock < pCommitter->dReader.mBlock.nItem) { tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); } else { pBlock = NULL; } + code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid, pTbData->maxSkmVer); + if (code) goto _err; + tMapDataReset(&pCommitter->dWriter.mBlock); + code = tBlockDataSetSchema(&pCommitter->dWriter.bData, pCommitter->skmTable.pTSchema); + if (code) goto _err; while (pBlock && pRow) { int32_t c = tBlockCmprFn(pBlock, &(SBlock){.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)}); if (c < 0) { // disk @@ -959,8 +941,8 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { } else { pBlock = NULL; } - } else if (c < 0) { // memory - code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey, 1); + } else if (c > 0) { // memory + code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey); if (code) goto _err; // next @@ -977,7 +959,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { code = tsdbMergeAsSubBlock(pCommitter, pIter, pBlock); if (code) goto _err; } else { - // code = tsdbMergeTableData(pCommitter, pIter, pBlock, NULL, 1); + code = tsdbMergeCommitData(pCommitter, pIter, pBlock); if (code) goto _err; } @@ -999,6 +981,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); if (code) goto _err; + // next iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); @@ -1008,8 +991,11 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { } // merge with last - int32_t nRowLeft = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); - if (pCommitter->dReader.pRowInfo) { + code = tsdbMergeCommitLast(pCommitter, pIter); + if (code) goto _err; +#if 0 + int32_t nRowLeft = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = + VERSION_MIN}); if (pCommitter->dReader.pRowInfo) { for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { int64_t uid = pCommitter->dReader.bDatal.aUid[iRow]; if (uid == pTbData->uid) { @@ -1022,6 +1008,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { code = tsdbMergeCommitLast(pCommitter, pIter, &nRowLeft); if (code) goto _err; } +#endif // end if (pCommitter->dWriter.mBlock.nItem > 0) { @@ -1051,14 +1038,14 @@ _err: static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; - // write aBlockL - code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL, NULL); - if (code) goto _err; - // write aBlockIdx code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx, NULL); if (code) goto _err; + // write aBlockL + code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL, NULL); + if (code) goto _err; + // update file header code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter); if (code) goto _err; @@ -1157,7 +1144,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pCommitter->dReader.pRowInfo->row, NULL); if (code) goto _err; - code = tsdbCommitNextLastRow(pCommitter); + code = tsdbCommitterNextLastRow(pCommitter); if (code) goto _err; nRow--; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 79929c44a0..9634b14bf7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -537,7 +537,7 @@ _err: int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf) { int32_t code = 0; int64_t offset = pReader->pSet->pHeadF->offset; - int64_t size = pReader->pSet->pHeadF->size - offset; + int64_t size = pReader->pSet->pHeadF->loffset - offset; uint8_t *pBuf = NULL; int64_t n; uint32_t delimiter; @@ -604,7 +604,7 @@ _err: int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) { int32_t code = 0; int64_t offset = pReader->pSet->pHeadF->loffset; - int64_t size = pReader->pSet->pHeadF->offset - offset; + int64_t size = pReader->pSet->pHeadF->size - offset; int64_t n; uint32_t delimiter; uint8_t *pBuf = NULL; From e6d374b0066eb0011fab2f298453d71ee6caaa69 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 3 Aug 2022 12:19:13 +0000 Subject: [PATCH 006/194] make it compile --- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 9634b14bf7..8417b0d1b8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -729,6 +729,21 @@ _err: return code; } +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2) { + int32_t code = 0; + // TODO + return code; +} + +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2) { + int32_t code = 0; + ASSERT(0); + // TODO + return code; +} + static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SSubBlock *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; int64_t size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); @@ -2098,6 +2113,20 @@ _err: return code; } +int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, + uint8_t **ppBuf2, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + +int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, + uint8_t **ppBuf2, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int32_t code = 0; int64_t n; From d1c7e3a531cffe060c49eee955d9da4e75c6a456 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 4 Aug 2022 05:42:15 +0000 Subject: [PATCH 007/194] more last refact --- source/dnode/vnode/src/inc/tsdb.h | 29 +++-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 25 +++-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 104 +++++++++--------- 4 files changed, 90 insertions(+), 72 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4c31b3e07d..29626ed67d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -134,14 +134,15 @@ int32_t tGetColData(uint8_t *p, SColData *pColData); #define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) #define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) #define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) -int32_t tBlockDataInit(SBlockData *pBlockData); -void tBlockDataReset(SBlockData *pBlockData); -int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema); +int32_t tBlockDataInit(SBlockData *pBlockData); +void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear); +void tBlockDataReset(SBlockData *pBlockData); +int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid); +int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); +int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); + int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom); void tBlockDataClearData(SBlockData *pBlockData); -void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear); -int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); -int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema); int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); @@ -468,13 +469,17 @@ struct SColData { uint8_t *pData; }; +// (SBlockData){.suid = 0, .uid = 0}: block data not initialized +// (SBlockData){.suid = suid, .uid = uid}: block data for ONE child table int .data file +// (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file +// (SBlockData){.suid = 0, .uid = uid}: block data for 1 normal table int .last/.data file struct SBlockData { - int64_t suid; // 0 means normal table data block - int64_t uid; // 0 means block data in .last file, others in .data file - int32_t nRow; - int64_t *aUid; - int64_t *aVersion; - TSKEY *aTSKEY; + int64_t suid; // 0 means normal table block data, otherwise child table block data + int64_t uid; // 0 means block data in .last file, otherwise in .data file + int32_t nRow; // number of rows + int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) + int64_t *aVersion; // versions of each row + TSKEY *aTSKEY; // timestamp of each row SArray *aIdx; // SArray SArray *aColData; // SArray }; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 63aa5f3ec8..3f38b664c2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -296,22 +296,28 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { ASSERT(pCommitter->dReader.pReader); ASSERT(pCommitter->dReader.pRowInfo); + SBlockData *pBlockDatal = &pCommitter->dReader.bDatal; pCommitter->dReader.iRow++; - if (pCommitter->dReader.iRow < pCommitter->dReader.bDatal.nRow) { - pCommitter->dReader.pRowInfo->uid = pCommitter->dReader.bData.aUid[pCommitter->dReader.iRow]; - pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + if (pCommitter->dReader.iRow < pBlockDatal->nRow) { + if (pBlockDatal->uid == 0) { + pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[pCommitter->dReader.iRow]; + } + pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); } else { pCommitter->dReader.iBlockL++; if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, &pCommitter->dReader.bDatal, - NULL, NULL); + code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal, NULL, NULL); if (code) goto _exit; pCommitter->dReader.iRow = 0; - pCommitter->dReader.pRowInfo->suid = pCommitter->dReader.pBlockL->suid; - pCommitter->dReader.pRowInfo->uid = pCommitter->dReader.bData.aUid[pCommitter->dReader.iRow]; - pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(&pCommitter->dReader.bDatal, pCommitter->dReader.iRow); + pCommitter->dReader.pRowInfo->suid = pBlockDatal->suid; + if (pBlockDatal->uid) { + pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; + } else { + pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[0]; + } + pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); } else { pCommitter->dReader.pRowInfo = NULL; } @@ -354,15 +360,16 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { } else { pCommitter->dReader.pBlockIdx = NULL; } + tBlockDataReset(&pCommitter->dReader.bData); // last code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL, NULL); if (code) goto _err; pCommitter->dReader.iBlockL = -1; - pCommitter->dReader.bDatal.nRow = 0; pCommitter->dReader.iRow = -1; pCommitter->dReader.pRowInfo = &pCommitter->dReader.rowInfo; + tBlockDataReset(&pCommitter->dReader.bDatal); code = tsdbCommitterNextLastRow(pCommitter); if (code) goto _err; } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8417b0d1b8..73bac77e6d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -591,8 +591,8 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB ASSERT(n + sizeof(TSCKSUM) == size); -_exit: tFree(pBuf); +_exit: return code; _err: @@ -658,8 +658,8 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) ASSERT(n + sizeof(TSCKSUM) == size); -_exit: tFree(pBuf); +_exit: return code; _err: diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 205d8b29dc..7beac23912 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1121,6 +1121,8 @@ static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) { int32_t tBlockDataInit(SBlockData *pBlockData) { int32_t code = 0; + pBlockData->suid = 0; + pBlockData->uid = 0; pBlockData->nRow = 0; pBlockData->aUid = NULL; pBlockData->aVersion = NULL; @@ -1141,36 +1143,45 @@ _exit: return code; } -void tBlockDataReset(SBlockData *pBlockData) { - pBlockData->nRow = 0; - taosArrayClear(pBlockData->aIdx); -} - void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear) { tFree((uint8_t *)pBlockData->aUid); tFree((uint8_t *)pBlockData->aVersion); tFree((uint8_t *)pBlockData->aTSKEY); taosArrayDestroy(pBlockData->aIdx); taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL); - pBlockData->aColData = NULL; - pBlockData->aIdx = NULL; - pBlockData->aTSKEY = NULL; + pBlockData->aUid = NULL; pBlockData->aVersion = NULL; + pBlockData->aTSKEY = NULL; + pBlockData->aIdx = NULL; + pBlockData->aColData = NULL; } -int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema) { - int32_t code = 0; - SColData *pColData; - STColumn *pTColumn; +void tBlockDataReset(SBlockData *pBlockData) { + pBlockData->suid = 0; + pBlockData->uid = 0; + pBlockData->nRow = 0; + taosArrayClear(pBlockData->aIdx); +} + +int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid) { + int32_t code = 0; + + ASSERT(suid || uid); tBlockDataReset(pBlockData); - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - pTColumn = &pTSchema->columns[iColumn]; + pBlockData->suid = suid; + pBlockData->uid = uid; - code = tBlockDataAddColData(pBlockData, iColumn - 1, &pColData); - if (code) goto _exit; + if (pTSchema) { + for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { + STColumn *pTColumn = &pTSchema->columns[iColumn]; + SColData *pColData; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) != 0); + code = tBlockDataAddColData(pBlockData, iColumn - 1, &pColData); + if (code) goto _exit; + + tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) != 0); + } } _exit: @@ -1211,52 +1222,47 @@ _err: return code; } -int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) { int32_t code = 0; - // TSDBKEY + ASSERT(pBlockData->suid || pBlockData->uid); + + // uid + if (pBlockData->uid == 0) { + ASSERT(uid); + code = tRealloc((uint8_t **)&pBlockData->aUid, sizeof(int64_t) * (pBlockData->nRow + 1)); + if (code) goto _err; + pBlockData->aUid[pBlockData->nRow] = uid; + } + // version code = tRealloc((uint8_t **)&pBlockData->aVersion, sizeof(int64_t) * (pBlockData->nRow + 1)); if (code) goto _err; + pBlockData->aVersion[pBlockData->nRow] = TSDBROW_VERSION(pRow); + // timestamp code = tRealloc((uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * (pBlockData->nRow + 1)); if (code) goto _err; - pBlockData->aVersion[pBlockData->nRow] = TSDBROW_VERSION(pRow); pBlockData->aTSKEY[pBlockData->nRow] = TSDBROW_TS(pRow); // OTHER - int32_t iColData = 0; - int32_t nColData = taosArrayGetSize(pBlockData->aIdx); - SRowIter iter = {0}; - SRowIter *pIter = &iter; - SColData *pColData; - SColVal *pColVal; + SRowIter rIter = {0}; + SColVal *pColVal; - if (nColData == 0) goto _exit; + tRowIterInit(&rIter, pRow, pTSchema); + pColVal = tRowIterNext(&rIter); + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - tRowIterInit(pIter, pRow, pTSchema); - pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - pColVal = tRowIterNext(pIter); + while (pColVal && pColVal->cid < pColData->cid) { + pColVal = tRowIterNext(&rIter); + } - while (pColData) { - if (pColVal) { - if (pColData->cid == pColVal->cid) { - code = tColDataAppendValue(pColData, pColVal); - if (code) goto _err; - - pColVal = tRowIterNext(pIter); - pColData = ((++iColData) < nColData) ? tBlockDataGetColDataByIdx(pBlockData, iColData) : NULL; - } else if (pColData->cid < pColVal->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _err; - - pColData = ((++iColData) < nColData) ? tBlockDataGetColDataByIdx(pBlockData, iColData) : NULL; - } else { - pColVal = tRowIterNext(pIter); - } - } else { + if (pColVal == NULL || pColVal->cid > pColData->cid) { code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); if (code) goto _err; - - pColData = ((++iColData) < nColData) ? tBlockDataGetColDataByIdx(pBlockData, iColData) : NULL; + } else { + code = tColDataAppendValue(pColData, pColVal); + if (code) goto _err; + pColVal = tRowIterNext(&rIter); } } From 67db73b56003764532c53bb5553cd81474252675 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 4 Aug 2022 07:58:53 +0000 Subject: [PATCH 008/194] more last refact work --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 296 +++++++++++------------ 2 files changed, 136 insertions(+), 162 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 29626ed67d..4c7dbd0a45 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -140,9 +140,9 @@ void tBlockDataReset(SBlockData *pBlockData); int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid); int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +void tBlockDataClearData(SBlockData *pBlockData); int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom); -void tBlockDataClearData(SBlockData *pBlockData); int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3f38b664c2..e5c6ee44a0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -299,7 +299,9 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { SBlockData *pBlockDatal = &pCommitter->dReader.bDatal; pCommitter->dReader.iRow++; if (pCommitter->dReader.iRow < pBlockDatal->nRow) { - if (pBlockDatal->uid == 0) { + if (pBlockDatal->uid) { + pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; + } else { pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[pCommitter->dReader.iRow]; } pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); @@ -327,6 +329,28 @@ _exit: return code; } +static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { + int32_t code = 0; + + ASSERT(pCommitter->dReader.pBlockIdx); + + pCommitter->dReader.iBlockIdx++; + if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { + pCommitter->dReader.pBlockIdx = + (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); + + code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); + if (code) goto _exit; + + ASSERT(pCommitter->dReader.mBlock.nItem > 0); + } else { + pCommitter->dReader.pBlockIdx = NULL; + } + +_exit: + return code; +} + static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; @@ -469,37 +493,18 @@ _exit: return code; } -static int32_t tsdbCommitBlockData(SCommitter *pCommitter, SBlockData *pBlockData, SBlock *pBlock, SBlockIdx *pBlockIdx, - int8_t toDataOnly) { - int32_t code = 0; - - if (pBlock->nSubBlock == 0) { - if (!toDataOnly && pBlockData->nRow < pCommitter->minRow) { - pBlock->last = 1; - } else { - pBlock->last = 0; - } - } - - code = - tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, NULL, NULL, pBlockIdx, pBlock, pCommitter->cmprAlg); - if (code) goto _err; - - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); - if (code) goto _err; - - return code; - -_err: - return code; -} - -static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) { +static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { int32_t code = 0; SBlock block; ASSERT(pCommitter->dWriter.bData.nRow > 0); + if (pBlock) { + block = *pBlock; // as a subblock + } else { + tBlockReset(&block); // as a new block + } + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bData, &block, NULL, NULL, pCommitter->cmprAlg); if (code) goto _exit; @@ -556,14 +561,14 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow1)); if (code) goto _err; - code = tBlockDataAppendRow(pBlockDataW, pRow1, pCommitter->skmRow.pTSchema); + code = tBlockDataAppendRow(pBlockDataW, pRow1, pCommitter->skmRow.pTSchema, pTbData->uid); if (code) goto _err; // next tsdbTbDataIterNext(pIter); pRow1 = tsdbTbDataIterGet(pIter); } else if (c > 0) { - code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL); + code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid); if (code) goto _err; iRow++; @@ -578,13 +583,13 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S // check if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter); + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; } } while (pRow2) { - code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL); + code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid); if (code) goto _err; iRow++; @@ -596,14 +601,14 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S // check if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter); + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; } } // check if (pBlockDataW->nRow > 0) { - code = tsdbCommitDataBlock(pCommitter); + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; } @@ -620,7 +625,6 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter SBlockData *pBlockData = &pCommitter->dWriter.bData; tBlockDataClearData(pBlockData); - TSDBROW *pRow = tsdbTbDataIterGet(pIter); while (true) { if (pRow == NULL) { @@ -636,7 +640,7 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter if (code) goto _err; // append - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); if (code) goto _err; tsdbTbDataIterNext(pIter); @@ -650,7 +654,7 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { _write_block: - code = tsdbCommitDataBlock(pCommitter); + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; } } @@ -662,29 +666,6 @@ _err: return code; } -static int32_t tsdbCommitTableDiskData(SCommitter *pCommitter, SBlock *pBlock, SBlockIdx *pBlockIdx) { - int32_t code = 0; - SBlock block; - - if (pBlock->last) { - code = tsdbReadBlockData(pCommitter->dReader.pReader, pBlockIdx, pBlock, &pCommitter->dReader.bData, NULL, NULL); - if (code) goto _err; - - tBlockReset(&block); - code = tsdbCommitBlockData(pCommitter, &pCommitter->dReader.bData, &block, pBlockIdx, 0); - if (code) goto _err; - } else { - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); - if (code) goto _err; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb commit table disk data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} - static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) { int32_t nRow = 0; @@ -713,31 +694,29 @@ static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, S SBlockData *pBlockData = &pCommitter->dWriter.bData; tBlockDataClearData(pBlockData); + TSDBROW *pRow = tsdbTbDataIterGet(pIter); while (true) { - TSDBROW *pRow = tsdbTbDataIterGet(pIter); + if (pRow == NULL) break; + + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); + if (code) goto _err; + + tsdbTbDataIterNext(pIter); + pRow = tsdbTbDataIterGet(pIter); if (pRow) { TSDBKEY rowKey = TSDBROW_KEY(pRow); if (tsdbKeyCmprFn(&rowKey, &pBlock->maxKey) > 0) { pRow = NULL; } } - - if (pRow == NULL) { - break; - } - - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; - - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); - if (code) goto _err; } - SBlock block = *pBlock; - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &block, NULL, NULL, pCommitter->cmprAlg); - if (code) goto _err; + ASSERT(pBlockData->nRow > 0 && pBlock->nRow + pBlockData->nRow <= pCommitter->maxRow); - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); + code = tsdbCommitDataBlock(pCommitter, pBlock); if (code) goto _err; return code; @@ -753,36 +732,51 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { int32_t nRow = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); if (pCommitter->dReader.pRowInfo) { - for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { - if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break; - nRow++; + if (pCommitter->dReader.pRowInfo->suid) { + for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { + if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break; + nRow++; + } + } else { + ASSERT(pCommitter->dReader.iRow == 0); + nRow += pCommitter->dReader.bDatal.nRow; } } if (nRow == 0) goto _exit; - SBlockData *pBlockData; - TSDBROW *pRow = tsdbTbDataIterGet(pIter); - SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; - + TSDBROW *pRow = tsdbTbDataIterGet(pIter); if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { pRow = NULL; } - if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; + if (pRowInfo && pRowInfo->uid != pTbData->uid) { pRowInfo = NULL; } while (nRow) { + SBlockData *pBlockData; + int8_t toData; + if (nRow < pCommitter->minRow) { // to .last + toData = 0; pBlockData = &pCommitter->dWriter.bDatal; - // check if same schema - if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; + // commit and reset block data schema if need + if (pBlockData->nRow > 0) { + if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; - code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema); + tBlockDataReset(pBlockData); + } + } + + // set block data schema if need + if (pBlockData->suid == 0 && pBlockData->uid == 0) { + code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema, pTbData->suid, + pTbData->suid ? 0 : pTbData->uid); if (code) goto _err; } @@ -791,7 +785,9 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { if (code) goto _err; } } else { // to .data + toData = 1; pBlockData = &pCommitter->dWriter.bData; + ASSERT(pBlockData->nRow == 0); } while (pRow && pRowInfo) { @@ -800,7 +796,7 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); if (code) goto _err; - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); if (code) goto _err; tsdbTbDataIterNext(pIter); @@ -809,14 +805,14 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { pRow = NULL; } } else if (c > 0) { - code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL); + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid); if (code) goto _err; code = tsdbCommitterNextLastRow(pCommitter); if (code) goto _err; pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + if (pRowInfo && pRowInfo->uid != pTbData->uid) { pRowInfo = NULL; } } else { @@ -824,16 +820,12 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { } nRow--; - - if (pBlockData->uid) { // .data block - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter); + if (toData) { + if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; - goto _outer_break; } - } else { - ASSERT(pBlockData->nRow <= pCommitter->maxRow); } } @@ -841,7 +833,7 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); if (code) goto _err; - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); if (code) goto _err; tsdbTbDataIterNext(pIter); @@ -851,40 +843,34 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { } nRow--; - if (pBlockData->uid) { // .data block - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter); + if (toData) { + if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; - goto _outer_break; } - } else { - ASSERT(pBlockData->nRow <= pCommitter->maxRow); } } while (pRowInfo) { - code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL); + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid); if (code) goto _err; code = tsdbCommitterNextLastRow(pCommitter); if (code) goto _err; pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && (pRowInfo->suid != pTbData->suid || pRowInfo->uid != pTbData->uid)) { + if (pRowInfo && pRowInfo->uid != pTbData->uid) { pRowInfo = NULL; } nRow--; - if (pBlockData->uid) { // .data block - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter); + if (toData) { + if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter, NULL); if (code) goto _err; - goto _outer_break; } - } else { - ASSERT(pBlockData->nRow <= pCommitter->maxRow); } } @@ -922,8 +908,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { int32_t iBlock = 0; SBlock block; SBlock *pBlock = █ - if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0 && - iBlock < pCommitter->dReader.mBlock.nItem) { + if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); } else { pBlock = NULL; @@ -933,8 +918,10 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { if (code) goto _err; tMapDataReset(&pCommitter->dWriter.mBlock); - code = tBlockDataSetSchema(&pCommitter->dWriter.bData, pCommitter->skmTable.pTSchema); + code = tBlockDataSetSchema(&pCommitter->dWriter.bData, pCommitter->skmTable.pTSchema, pTbData->suid, pTbData->uid); if (code) goto _err; + + // .data merge while (pBlock && pRow) { int32_t c = tBlockCmprFn(pBlock, &(SBlock){.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)}); if (c < 0) { // disk @@ -997,25 +984,9 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { } } - // merge with last + // .data append and .last merge code = tsdbMergeCommitLast(pCommitter, pIter); if (code) goto _err; -#if 0 - int32_t nRowLeft = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = - VERSION_MIN}); if (pCommitter->dReader.pRowInfo) { - for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { - int64_t uid = pCommitter->dReader.bDatal.aUid[iRow]; - if (uid == pTbData->uid) { - nRowLeft++; - } - } - } - - while (nRowLeft) { - code = tsdbMergeCommitLast(pCommitter, pIter, &nRowLeft); - if (code) goto _err; - } -#endif // end if (pCommitter->dWriter.mBlock.nItem > 0) { @@ -1081,7 +1052,7 @@ _err: static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { int32_t code = 0; - // data + // .data while (true) { if (pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) >= 0) break; @@ -1094,61 +1065,64 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { goto _err; } - pCommitter->dReader.iBlockIdx++; - if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { - pCommitter->dReader.pBlockIdx = - (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - - code = - tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); - if (code) goto _err; - } else { - pCommitter->dReader.pBlockIdx = NULL; - } + code = tsdbCommitterNextTableData(pCommitter); + if (code) goto _err; } - // last + // .last while (true) { if (pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, &toTable) >= 0) break; - // commit if not same schema - if (pCommitter->dWriter.bDatal.nRow > 0) { - if (pCommitter->dWriter.bDatal.suid != pCommitter->dReader.pRowInfo->suid || - pCommitter->dWriter.bDatal.suid == 0) { + SBlockData *pBlockDataR = &pCommitter->dReader.bDatal; + SBlockData *pBlockDataW = &pCommitter->dWriter.bDatal; + tb_uid_t suid = pCommitter->dReader.pRowInfo->suid; + tb_uid_t uid = pCommitter->dReader.pRowInfo->uid; + + ASSERT((pBlockDataR->suid && !pBlockDataR->uid) || (!pBlockDataR->suid && pBlockDataR->uid)); + ASSERT(pBlockDataR->nRow > 0); + + // commit and reset block data schema if need + if (pBlockDataW->nRow > 0) { + if (pBlockDataW->suid != pCommitter->dReader.pRowInfo->suid || pBlockDataW->suid == 0) { code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; + + tBlockDataReset(pBlockDataW); } } - if (pCommitter->dWriter.bDatal.nRow == 0) { - code = tsdbCommitterUpdateTableSchema(pCommitter, pCommitter->dReader.pRowInfo->suid, - pCommitter->dReader.pRowInfo->suid, 1 /*TODO*/); + // set block data schema if need + if (pBlockDataW->suid == 0 && pBlockDataW->uid == 0) { + code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid, 1 /*TOOD*/); if (code) goto _err; - pCommitter->dWriter.bDatal.suid = pCommitter->dReader.pRowInfo->suid; - code = tBlockDataSetSchema(&pCommitter->dWriter.bDatal, pCommitter->skmTable.pTSchema); + code = tBlockDataSetSchema(pBlockDataW, pCommitter->skmTable.pTSchema, suid, suid ? 0 : uid); if (code) goto _err; } // check if it can make sure that one table data in one block - int64_t uid = pCommitter->dReader.pRowInfo->uid; int32_t nRow = 0; - for (int32_t iRow = pCommitter->dReader.iRow; - (iRow < pCommitter->dReader.bDatal.nRow) && (pCommitter->dReader.bDatal.aUid[iRow] == uid); iRow++) { - nRow++; + if (pBlockDataR->suid) { + for (int32_t iRow = pCommitter->dReader.iRow; (iRow < pBlockDataR->nRow) && (pBlockDataR->aUid[iRow] == uid); + iRow++) { + nRow++; + } + } else { + ASSERT(pCommitter->dReader.iRow == 0); + nRow = pBlockDataR->nRow; } ASSERT(nRow > 0 && nRow < pCommitter->minRow); - if (pCommitter->dWriter.bDatal.nRow + nRow > pCommitter->maxRow) { - ASSERT(pCommitter->dWriter.bDatal.nRow > 0); + if (pBlockDataW->nRow + nRow > pCommitter->maxRow) { + ASSERT(pBlockDataW->nRow > 0); code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; } while (nRow > 0) { - code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pCommitter->dReader.pRowInfo->row, NULL); + code = tBlockDataAppendRow(pBlockDataW, &pCommitter->dReader.pRowInfo->row, NULL, uid); if (code) goto _err; code = tsdbCommitterNextLastRow(pCommitter); From 03a615c01adb5575aa4a191955672035cbb3792b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 4 Aug 2022 08:04:34 +0000 Subject: [PATCH 009/194] more --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 32 ++++++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a04d18aae2..eec7a3e4a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -41,6 +41,7 @@ struct STsdbSnapReader { SArray* aDelData; // SArray }; +#if 0 static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; STsdb* pTsdb = pReader->pTsdb; @@ -245,9 +246,11 @@ _err: tstrerror(code)); return code; } +#endif int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) { - int32_t code = 0; + int32_t code = 0; +#if 0 STsdbSnapReader* pReader = NULL; // alloc @@ -310,11 +313,13 @@ _err: tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); *ppReader = NULL; +#endif return code; } int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { - int32_t code = 0; + int32_t code = 0; +#if 0 STsdbSnapReader* pReader = *ppReader; if (pReader->pDataFReader) { @@ -337,11 +342,13 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { taosMemoryFree(pReader); *ppReader = NULL; +#endif return code; } int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; +#if 0 *ppData = NULL; @@ -380,6 +387,7 @@ _exit: _err: tsdbError("vgId:%d, vnode snapshot tsdb read for %s failed since %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path, tstrerror(code)); +#endif return code; } @@ -430,6 +438,7 @@ struct STsdbSnapWriter { SArray* aDelIdxW; }; +#if 0 static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; int32_t iRow = 0; // todo @@ -808,7 +817,8 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { if (code) goto _err; _exit: - tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); + tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode), + pWriter->pTsdb->path); return code; _err: @@ -1036,7 +1046,9 @@ _err: tstrerror(code)); return code; } +#endif +#if 0 static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; @@ -1082,9 +1094,11 @@ _err: tstrerror(code)); return code; } +#endif int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { - int32_t code = 0; + int32_t code = 0; +#if 0 STsdbSnapWriter* pWriter = NULL; // alloc @@ -1153,11 +1167,13 @@ _err: tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); *ppWriter = NULL; +#endif return code; } int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { - int32_t code = 0; + int32_t code = 0; +#if 0 STsdbSnapWriter* pWriter = *ppWriter; if (rollback) { @@ -1188,11 +1204,13 @@ _err: pWriter->pTsdb->path, tstrerror(code)); taosMemoryFree(pWriter); *ppWriter = NULL; +#endif return code; } int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; + int32_t code = 0; +#if 0 SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; // ts data @@ -1216,10 +1234,12 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) _exit: tsdbDebug("vgId:%d, tsdb snapshot write for %s succeed", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); + return code; _err: tsdbError("vgId:%d, tsdb snapshot write for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path, tstrerror(code)); +#endif return code; } From 2cb114caad617598790ed0690773d397a9d6003e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 4 Aug 2022 10:15:51 +0000 Subject: [PATCH 010/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 33 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 363 ++++++++++-------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 27 +- 3 files changed, 217 insertions(+), 206 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4c7dbd0a45..32037cbf19 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -230,16 +230,12 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBuf, SBlockIdx *pBlockIdx); int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf); -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2, - SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg); - -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); - -/* new */ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, uint8_t **ppBuf2, int8_t cmprAlg); int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, uint8_t **ppBuf2, int8_t cmprAlg); + +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); @@ -251,8 +247,6 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf); - -/* new */ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, @@ -401,12 +395,12 @@ typedef struct { int16_t cid; int8_t type; int8_t smaOn; - int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE - int32_t offset; + int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE + int32_t szOrigin; // original column value size (only save for variant data type) int32_t szBitmap; // bitmap size int32_t szOffset; // size of offset, only for variant-length data type int32_t szValue; // compressed column value size - int32_t szOrigin; // original column value size (only save for variant data type) + int32_t offset; } SBlockCol; typedef struct { @@ -427,7 +421,6 @@ struct SBlock { int64_t minVersion; int64_t maxVersion; int32_t nRow; - int8_t last; int8_t hasDup; int8_t nSubBlock; SSubBlock aSubBlock[TSDB_MAX_SUBBLOCKS]; @@ -435,26 +428,18 @@ struct SBlock { struct SBlockL { int64_t suid; - struct { - int64_t uid; - int64_t version; - TSKEY ts; - } minKey; - struct { - int64_t uid; - int64_t version; - TSKEY ts; - } maxKey; + int64_t minUid; + int64_t maxUid; int64_t minVer; int64_t maxVer; int32_t nRow; - int8_t cmprAlg; int64_t offset; - int32_t szBlock; + int8_t cmprAlg; int32_t szBlockCol; int32_t szUid; int32_t szVer; int32_t szTSKEY; + int32_t szBlock; }; struct SColData { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 73bac77e6d..2776ac559e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1765,91 +1765,56 @@ static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { pBlock->nRow += pBlockData->nRow; } -static int32_t tsdbWriteBlockDataKey(SSubBlock *pSubBlock, SBlockData *pBlockData, uint8_t **ppBuf1, int64_t *nDataP, - uint8_t **ppBuf2) { +static int32_t tsdbWriteDataArray(uint8_t *aData, int32_t nEle, int8_t type, int8_t cmprAlg, int32_t *rSize, + uint8_t **ppBuf1, int64_t nBuf1, uint8_t **ppBuf2) { int32_t code = 0; - int64_t size; - int64_t tsize; + int32_t size; - if (pSubBlock->cmprAlg == NO_COMPRESSION) { - pSubBlock->szVersion = sizeof(int64_t) * pSubBlock->nRow; - pSubBlock->szTSKEY = sizeof(TSKEY) * pSubBlock->nRow; - - code = tRealloc(ppBuf1, *nDataP + pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM)); - if (code) goto _err; - - // VERSION - memcpy(*ppBuf1 + *nDataP, pBlockData->aVersion, pSubBlock->szVersion); - - // TSKEY - memcpy(*ppBuf1 + *nDataP + pSubBlock->szVersion, pBlockData->aTSKEY, pSubBlock->szTSKEY); + if (IS_VAR_DATA_TYPE(type)) { + size = nEle; } else { - size = (sizeof(int64_t) + sizeof(TSKEY)) * pSubBlock->nRow + COMP_OVERFLOW_BYTES * 2; - - code = tRealloc(ppBuf1, *nDataP + size + sizeof(TSCKSUM)); - if (code) goto _err; - - tsize = sizeof(int64_t) * pSubBlock->nRow + COMP_OVERFLOW_BYTES; - if (pSubBlock->cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf2, tsize); - if (code) goto _err; - } - - // VERSION - pSubBlock->szVersion = - tsCompressBigint((char *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, pBlockData->nRow, - *ppBuf1 + *nDataP, size, pSubBlock->cmprAlg, *ppBuf2, tsize); - if (pSubBlock->szVersion <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - // TSKEY - pSubBlock->szTSKEY = tsCompressTimestamp((char *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, - pBlockData->nRow, *ppBuf1 + *nDataP + pSubBlock->szVersion, - size - pSubBlock->szVersion, pSubBlock->cmprAlg, *ppBuf2, tsize); - if (pSubBlock->szTSKEY <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - ASSERT(pSubBlock->szVersion + pSubBlock->szTSKEY <= size); + size = tDataTypes[type].bytes * nEle; } - // checksum - size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); - taosCalcChecksumAppend(0, *ppBuf1 + *nDataP, size); + if (cmprAlg == NO_COMPRESSION) { + code = tRealloc(ppBuf1, nBuf1 + size); + if (code) goto _exit; - *nDataP += size; - return code; + memcpy(*ppBuf1 + nBuf1, aData, size); + *rSize = size; + } else { + code = tRealloc(ppBuf1, size + COMP_OVERFLOW_BYTES); + if (code) goto _exit; -_err: + if (cmprAlg == TWO_STAGE_COMP) { + code = tRealloc(ppBuf2, size + COMP_OVERFLOW_BYTES); + if (code) goto _exit; + } + + int32_t n = tDataTypes[type].compFunc(aData, tDataTypes[type].bytes * nEle, nEle, *ppBuf1 + nBuf1, + size + COMP_OVERFLOW_BYTES, cmprAlg, *ppBuf2, size + COMP_OVERFLOW_BYTES); + if (n <= 0) { + code = TSDB_CODE_COMPRESS_ERROR; + goto _exit; + } + *rSize = n; + } + +_exit: return code; } -static int32_t tsdbWriteColData(SColData *pColData, SBlockCol *pBlockCol, SSubBlock *pSubBlock, uint8_t **ppBuf1, - int64_t *nDataP, uint8_t **ppBuf2) { +static int32_t tsdbWriteColData(SColData *pColData, SBlockCol *pBlockCol, int8_t cmprAlg, uint8_t **ppBuf1, + int64_t nBuf1, uint8_t **ppBuf2) { int32_t code = 0; int64_t size; int64_t n = 0; // BITMAP if (pColData->flag != HAS_VALUE) { - size = BIT2_SIZE(pColData->nVal) + COMP_OVERFLOW_BYTES; - - code = tRealloc(ppBuf1, *nDataP + n + size); + code = tsdbWriteDataArray(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, + &pBlockCol->szBitmap, ppBuf1, nBuf1 + n, ppBuf2); if (code) goto _err; - - code = tRealloc(ppBuf2, size); - if (code) goto _err; - - pBlockCol->szBitmap = - tsCompressTinyint((char *)pColData->pBitMap, BIT2_SIZE(pColData->nVal), BIT2_SIZE(pColData->nVal), - *ppBuf1 + *nDataP + n, size, TWO_STAGE_COMP, *ppBuf2, size); - if (pBlockCol->szBitmap <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } } else { pBlockCol->szBitmap = 0; } @@ -1857,60 +1822,29 @@ static int32_t tsdbWriteColData(SColData *pColData, SBlockCol *pBlockCol, SSubBl // OFFSET if (IS_VAR_DATA_TYPE(pColData->type)) { - size = sizeof(int32_t) * pColData->nVal + COMP_OVERFLOW_BYTES; - - code = tRealloc(ppBuf1, *nDataP + n + size); + code = tsdbWriteDataArray((uint8_t *)pColData->aOffset, pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, + &pBlockCol->szOffset, ppBuf1, nBuf1 + n, ppBuf2); if (code) goto _err; - - code = tRealloc(ppBuf2, size); - if (code) goto _err; - - pBlockCol->szOffset = tsCompressInt((char *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, pColData->nVal, - *ppBuf1 + *nDataP + n, size, TWO_STAGE_COMP, *ppBuf2, size); - if (pBlockCol->szOffset <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } } else { pBlockCol->szOffset = 0; } n += pBlockCol->szOffset; // VALUE - if (pSubBlock->cmprAlg == NO_COMPRESSION) { - pBlockCol->szValue = pColData->nData; - - code = tRealloc(ppBuf1, *nDataP + n + pBlockCol->szValue + sizeof(TSCKSUM)); + if (pColData->flag != (HAS_NULL | HAS_NONE)) { + code = tsdbWriteDataArray(pColData->pData, pColData->nData, pColData->type, cmprAlg, &pBlockCol->szValue, ppBuf1, + nBuf1 + n, ppBuf2); if (code) goto _err; - - memcpy(*ppBuf1 + *nDataP + n, pColData->pData, pBlockCol->szValue); } else { - size = pColData->nData + COMP_OVERFLOW_BYTES; - - code = tRealloc(ppBuf1, *nDataP + n + size + sizeof(TSCKSUM)); - if (code) goto _err; - - if (pSubBlock->cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf2, size); - if (code) goto _err; - } - - pBlockCol->szValue = - tDataTypes[pColData->type].compFunc((char *)pColData->pData, pColData->nData, pColData->nVal, - *ppBuf1 + *nDataP + n, size, pSubBlock->cmprAlg, *ppBuf2, size); - if (pBlockCol->szValue <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } + pBlockCol->szValue = 0; } n += pBlockCol->szValue; - pBlockCol->szOrigin = pColData->nData; // checksum n += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, *ppBuf1 + *nDataP, n); - - *nDataP += n; + code = tRealloc(ppBuf1, nBuf1 + n); + if (code) goto _err; + taosCalcChecksumAppend(0, *ppBuf1 + nBuf1, n); return code; @@ -1918,20 +1852,20 @@ _err: return code; } -static int32_t tsdbWriteBlockDataImpl(TdFilePtr pFD, SSubBlock *pSubBlock, SBlockDataHdr hdr, SArray *aBlockCol, - uint8_t *pData, int64_t nData, uint8_t **ppBuf) { +static int32_t tsdbWriteBlockDataImpl(TdFilePtr pFD, SBlockDataHdr hdr, SArray *aBlockCol, uint8_t *pData, + int64_t nData, uint8_t **ppBuf, int32_t *szBlockCol) { int32_t code = 0; int32_t nBlockCol = taosArrayGetSize(aBlockCol); int64_t size; int64_t n; // HDR + SArray - pSubBlock->szBlockCol = sizeof(hdr); + *szBlockCol = sizeof(hdr); for (int32_t iBlockCol = 0; iBlockCol < nBlockCol; iBlockCol++) { - pSubBlock->szBlockCol += tPutBlockCol(NULL, taosArrayGet(aBlockCol, iBlockCol)); + (*szBlockCol) += tPutBlockCol(NULL, taosArrayGet(aBlockCol, iBlockCol)); } - code = tRealloc(ppBuf, pSubBlock->szBlockCol + sizeof(TSCKSUM)); + code = tRealloc(ppBuf, *szBlockCol + sizeof(TSCKSUM)); if (code) goto _err; n = 0; @@ -1940,11 +1874,11 @@ static int32_t tsdbWriteBlockDataImpl(TdFilePtr pFD, SSubBlock *pSubBlock, SBloc for (int32_t iBlockCol = 0; iBlockCol < nBlockCol; iBlockCol++) { n += tPutBlockCol(*ppBuf + n, taosArrayGet(aBlockCol, iBlockCol)); } - taosCalcChecksumAppend(0, *ppBuf, pSubBlock->szBlockCol + sizeof(TSCKSUM)); + taosCalcChecksumAppend(0, *ppBuf, *szBlockCol + sizeof(TSCKSUM)); - ASSERT(n == pSubBlock->szBlockCol); + ASSERT(n == *szBlockCol); - n = taosWriteFile(pFD, *ppBuf, pSubBlock->szBlockCol + sizeof(TSCKSUM)); + n = taosWriteFile(pFD, *ppBuf, *szBlockCol + sizeof(TSCKSUM)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -2007,42 +1941,49 @@ _err: return code; } -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2, - SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg) { - int32_t code = 0; - SSubBlock *pSubBlock = &pBlock->aSubBlock[pBlock->nSubBlock++]; - SBlockCol blockCol; - SBlockCol *pBlockCol = &blockCol; - int64_t n; - TdFilePtr pFileFD = pBlock->last ? pWriter->pLastFD : pWriter->pDataFD; - SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; - uint8_t *p; - int64_t nData; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - SArray *aBlockCol = NULL; +int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, + uint8_t **ppBuf2, int8_t cmprAlg) { + int32_t code = 0; + + ASSERT((pBlockData->suid && pBlockData->uid) || (!pBlockData->suid && pBlockData->uid)); + + uint8_t *pBuf1 = NULL; + uint8_t *pBuf2 = NULL; if (!ppBuf1) ppBuf1 = &pBuf1; if (!ppBuf2) ppBuf2 = &pBuf2; tsdbUpdateBlockInfo(pBlockData, pBlock); + SSubBlock *pSubBlock = &pBlock->aSubBlock[pBlock->nSubBlock++]; + pSubBlock->nRow = pBlockData->nRow; pSubBlock->cmprAlg = cmprAlg; - if (pBlock->last) { - pSubBlock->offset = pWriter->fLast.size; - } else { - pSubBlock->offset = pWriter->fData.size; - } + pSubBlock->offset = pWriter->fData.size; // ======================= BLOCK DATA ======================= - // TSDBKEY - nData = 0; - code = tsdbWriteBlockDataKey(pSubBlock, pBlockData, ppBuf1, &nData, ppBuf2); + int64_t nBuf1 = 0; + + // VERSION + code = tsdbWriteDataArray((uint8_t *)pBlockData->aVersion, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pSubBlock->szVersion, ppBuf1, nBuf1, ppBuf2); if (code) goto _err; + nBuf1 += pSubBlock->szVersion; + + // TSKEY + code = tsdbWriteDataArray((uint8_t *)pBlockData->aTSKEY, pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg, + &pSubBlock->szTSKEY, ppBuf1, nBuf1, ppBuf2); + if (code) goto _err; + nBuf1 += pSubBlock->szTSKEY; + + // checksum + nBuf1 += sizeof(TSCKSUM); + code = tRealloc(ppBuf1, nBuf1); + if (code) goto _err; + taosCalcChecksumAppend(0, *ppBuf1, nBuf1); // COLUMNS - aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); + SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); if (aBlockCol == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -2050,46 +1991,46 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_ int32_t offset = 0; for (int32_t iCol = 0; iCol < taosArrayGetSize(pBlockData->aIdx); iCol++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); + SBlockCol blockCol = {0}; ASSERT(pColData->flag); if (pColData->flag == HAS_NONE) continue; - pBlockCol->cid = pColData->cid; - pBlockCol->type = pColData->type; - pBlockCol->smaOn = pColData->smaOn; - pBlockCol->flag = pColData->flag; + blockCol.cid = pColData->cid; + blockCol.type = pColData->type; + blockCol.smaOn = pColData->smaOn; + blockCol.flag = pColData->flag; + blockCol.szOrigin = pColData->nData; if (pColData->flag != HAS_NULL) { - code = tsdbWriteColData(pColData, pBlockCol, pSubBlock, ppBuf1, &nData, ppBuf2); + code = tsdbWriteColData(pColData, &blockCol, cmprAlg, ppBuf1, nBuf1, ppBuf2); if (code) goto _err; - pBlockCol->offset = offset; - offset = offset + pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); + blockCol.offset = offset; + offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } - if (taosArrayPush(aBlockCol, pBlockCol) == NULL) { + if (taosArrayPush(aBlockCol, &blockCol) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } } // write - code = tsdbWriteBlockDataImpl(pFileFD, pSubBlock, hdr, aBlockCol, *ppBuf1, nData, ppBuf2); + SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockData->suid, .uid = pBlockData->uid}; + code = tsdbWriteBlockDataImpl(pWriter->pDataFD, hdr, aBlockCol, *ppBuf1, nBuf1, ppBuf2, &pSubBlock->szBlockCol); if (code) goto _err; - pSubBlock->szBlock = pSubBlock->szBlockCol + sizeof(TSCKSUM) + nData; - if (pBlock->last) { - pWriter->fLast.size += pSubBlock->szBlock; - } else { - pWriter->fData.size += pSubBlock->szBlock; - } + pSubBlock->szBlock = pSubBlock->szBlockCol + sizeof(TSCKSUM) + nBuf1; + pWriter->fData.size += pSubBlock->szBlock; // ======================= BLOCK SMA ======================= pSubBlock->sOffset = 0; pSubBlock->nSma = 0; - if (pBlock->nSubBlock > 1 || pBlock->last || pBlock->hasDup) goto _exit; + if (pBlock->nSubBlock > 1 || pBlock->hasDup) goto _exit; code = tsdbWriteBlockSma(pWriter->pSmaFD, pBlockData, pSubBlock, ppBuf1); if (code) goto _err; @@ -2113,17 +2054,113 @@ _err: return code; } -int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, - uint8_t **ppBuf2, int8_t cmprAlg) { - int32_t code = 0; - // TODO - return code; -} - int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, uint8_t **ppBuf2, int8_t cmprAlg) { int32_t code = 0; - // TODO + + ASSERT((pBlockData->suid && !pBlockData->uid) || (!pBlockData->suid && pBlockData->uid)); + + uint8_t *pBuf1 = NULL; + uint8_t *pBuf2 = NULL; + + if (!ppBuf1) ppBuf1 = &pBuf1; + if (!ppBuf2) ppBuf2 = &pBuf2; + + pBlockL->suid = pBlockData->suid; + if (pBlockData->uid) { + pBlockL->maxUid = pBlockL->minUid = pBlockData->uid; + } else { + pBlockL->minUid = pBlockData->aUid[0]; + pBlockL->maxUid = pBlockData->aUid[pBlockData->nRow - 1]; + } + pBlockL->nRow = pBlockData->nRow; + pBlockL->offset = pWriter->fLast.size; + pBlockL->cmprAlg = cmprAlg; + pBlockL->minVer = VERSION_MAX; + pBlockL->maxVer = VERSION_MIN; + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + pBlockL->minVer = TMIN(pBlockL->minVer, pBlockData->aVersion[iRow]); + pBlockL->maxVer = TMAX(pBlockL->maxVer, pBlockData->aVersion[iRow]); + } + + // ======================= BLOCK DATA ======================= + int64_t nBuf1 = 0; + + // UID + if (pBlockData->uid == 0) { + code = tsdbWriteDataArray((uint8_t *)pBlockData->aUid, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pBlockL->szUid, ppBuf1, nBuf1, ppBuf2); + if (code) goto _err; + } else { + pBlockL->szUid = 0; + } + nBuf1 += pBlockL->szUid; + + // VERSION + code = tsdbWriteDataArray((uint8_t *)pBlockData->aVersion, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pBlockL->szVer, ppBuf1, nBuf1, ppBuf2); + if (code) goto _err; + nBuf1 += pBlockL->szVer; + + // TSKEY + code = tsdbWriteDataArray((uint8_t *)pBlockData->aTSKEY, pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg, + &pBlockL->szTSKEY, ppBuf1, nBuf1, ppBuf2); + if (code) goto _err; + nBuf1 += pBlockL->szTSKEY; + + // checksum + nBuf1 += sizeof(TSCKSUM); + code = tRealloc(ppBuf1, nBuf1); + if (code) goto _err; + taosCalcChecksumAppend(0, *ppBuf1, nBuf1); + + // COLUMNS + SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); + if (aBlockCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + int32_t offset = 0; + for (int32_t iCol = 0; iCol < taosArrayGetSize(pBlockData->aIdx); iCol++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); + SBlockCol blockCol = {0}; + + ASSERT(pColData->flag); + + if (pColData->flag == HAS_NONE) continue; + + blockCol.cid = pColData->cid; + blockCol.type = pColData->type; + blockCol.smaOn = pColData->smaOn; + blockCol.flag = pColData->flag; + blockCol.szOrigin = pColData->nData; + + if (pColData->flag != HAS_NULL) { + code = tsdbWriteColData(pColData, &blockCol, cmprAlg, ppBuf1, nBuf1, ppBuf2); + if (code) goto _err; + + blockCol.offset = offset; + offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + } + + if (taosArrayPush(aBlockCol, &blockCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + + // write + SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockData->suid, .uid = pBlockData->uid}; + code = tsdbWriteBlockDataImpl(pWriter->pLastFD, hdr, aBlockCol, *ppBuf1, nBuf1, ppBuf2, &pBlockL->szBlockCol); + if (code) goto _err; + + pBlockL->szBlock = pBlockL->szBlockCol + sizeof(TSCKSUM) + nBuf1; + pWriter->fLast.size += pBlockL->szBlock; + + return code; + +_err: return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 7beac23912..c17a7f4a09 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -230,7 +230,6 @@ int32_t tPutBlock(uint8_t *p, void *ph) { n += tPutI64v(p ? p + n : p, pBlock->minVersion); n += tPutI64v(p ? p + n : p, pBlock->maxVersion); n += tPutI32v(p ? p + n : p, pBlock->nRow); - n += tPutI8(p ? p + n : p, pBlock->last); n += tPutI8(p ? p + n : p, pBlock->hasDup); n += tPutI8(p ? p + n : p, pBlock->nSubBlock); for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { @@ -257,7 +256,6 @@ int32_t tGetBlock(uint8_t *p, void *ph) { n += tGetI64v(p + n, &pBlock->minVersion); n += tGetI64v(p + n, &pBlock->maxVersion); n += tGetI32v(p + n, &pBlock->nRow); - n += tGetI8(p + n, &pBlock->last); n += tGetI8(p + n, &pBlock->hasDup); n += tGetI8(p + n, &pBlock->nSubBlock); for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { @@ -290,7 +288,6 @@ int32_t tBlockCmprFn(const void *p1, const void *p2) { bool tBlockHasSma(SBlock *pBlock) { if (pBlock->nSubBlock > 1) return false; - if (pBlock->last) return false; if (pBlock->hasDup) return false; return pBlock->aSubBlock[0].nSma > 0; @@ -301,22 +298,18 @@ int32_t tPutBlockL(uint8_t *p, void *ph) { SBlockL *pBlockL = (SBlockL *)ph; n += tPutI64(p ? p + n : p, pBlockL->suid); - n += tPutI64(p ? p + n : p, pBlockL->minKey.uid); - n += tPutI64v(p ? p + n : p, pBlockL->minKey.version); - n += tPutI64(p ? p + n : p, pBlockL->minKey.ts); - n += tPutI64(p ? p + n : p, pBlockL->maxKey.uid); - n += tPutI64v(p ? p + n : p, pBlockL->maxKey.version); - n += tPutI64(p ? p + n : p, pBlockL->maxKey.ts); + n += tPutI64(p ? p + n : p, pBlockL->minUid); + n += tPutI64(p ? p + n : p, pBlockL->maxUid); n += tPutI64v(p ? p + n : p, pBlockL->minVer); n += tPutI64v(p ? p + n : p, pBlockL->maxVer); n += tPutI32v(p ? p + n : p, pBlockL->nRow); - n += tPutI8(p ? p + n : p, pBlockL->cmprAlg); n += tPutI64v(p ? p + n : p, pBlockL->offset); - n += tPutI32v(p ? p + n : p, pBlockL->szBlock); + n += tPutI8(p ? p + n : p, pBlockL->cmprAlg); n += tPutI32v(p ? p + n : p, pBlockL->szBlockCol); n += tPutI32v(p ? p + n : p, pBlockL->szUid); n += tPutI32v(p ? p + n : p, pBlockL->szVer); n += tPutI32v(p ? p + n : p, pBlockL->szTSKEY); + n += tPutI32v(p ? p + n : p, pBlockL->szBlock); return n; } @@ -326,22 +319,18 @@ int32_t tGetBlockL(uint8_t *p, void *ph) { SBlockL *pBlockL = (SBlockL *)ph; n += tGetI64(p + n, &pBlockL->suid); - n += tGetI64(p + n, &pBlockL->minKey.uid); - n += tGetI64v(p + n, &pBlockL->minKey.version); - n += tGetI64(p + n, &pBlockL->minKey.ts); - n += tGetI64(p + n, &pBlockL->maxKey.uid); - n += tGetI64v(p + n, &pBlockL->maxKey.version); - n += tGetI64(p + n, &pBlockL->maxKey.ts); + n += tGetI64(p + n, &pBlockL->minUid); + n += tGetI64(p + n, &pBlockL->maxUid); n += tGetI64v(p + n, &pBlockL->minVer); n += tGetI64v(p + n, &pBlockL->maxVer); n += tGetI32v(p + n, &pBlockL->nRow); - n += tGetI8(p + n, &pBlockL->cmprAlg); n += tGetI64v(p + n, &pBlockL->offset); - n += tGetI32v(p + n, &pBlockL->szBlock); + n += tGetI8(p + n, &pBlockL->cmprAlg); n += tGetI32v(p + n, &pBlockL->szBlockCol); n += tGetI32v(p + n, &pBlockL->szUid); n += tGetI32v(p + n, &pBlockL->szVer); n += tGetI32v(p + n, &pBlockL->szTSKEY); + n += tGetI32v(p + n, &pBlockL->szBlock); return n; } From d5c2d5941f4bf392225d18318b0115a3039cc557 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 5 Aug 2022 05:23:50 +0000 Subject: [PATCH 011/194] more work --- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 234 +++++++++++++++--- source/dnode/vnode/src/tsdb/tsdbUtil.c | 8 +- 2 files changed, 197 insertions(+), 45 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 2776ac559e..bc4656278c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -729,21 +729,6 @@ _err: return code; } -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2) { - int32_t code = 0; - // TODO - return code; -} - -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2) { - int32_t code = 0; - ASSERT(0); - // TODO - return code; -} - static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SSubBlock *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; int64_t size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); @@ -892,20 +877,25 @@ _err: return code; } -static int32_t tsdbReadBlockCol(SSubBlock *pSubBlock, uint8_t *p, SArray *aBlockCol) { +static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SBlockDataHdr *pHdr, SArray *aBlockCol) { int32_t code = 0; int32_t n = 0; SBlockCol blockCol; SBlockCol *pBlockCol = &blockCol; - if (!taosCheckChecksumWhole(p, pSubBlock->szBlockCol + sizeof(TSCKSUM))) { + // checksum + if (!taosCheckChecksumWhole(pBuf, szBlockCol + sizeof(TSCKSUM))) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } + // hdr + *pHdr = *(SBlockDataHdr *)pBuf; n += sizeof(SBlockDataHdr); - while (n < pSubBlock->szBlockCol) { - n += tGetBlockCol(p + n, pBlockCol); + + // aBlockCol + while (n < szBlockCol) { + n += tGetBlockCol(pBuf + n, pBlockCol); if (taosArrayPush(aBlockCol, pBlockCol) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -913,7 +903,7 @@ static int32_t tsdbReadBlockCol(SSubBlock *pSubBlock, uint8_t *p, SArray *aBlock } } - ASSERT(n == pSubBlock->szBlockCol); + ASSERT(n == szBlockCol); return code; @@ -921,10 +911,48 @@ _err: return code; } +static int32_t tsdbReadDataArray(uint8_t *pInput, int32_t szInput, int32_t nEle, int8_t type, int8_t cmprAlg, + uint8_t **ppOut, uint8_t **ppBuf) { + int32_t code = 0; + int32_t size; + + // size + if (IS_VAR_DATA_TYPE(type)) { + size = nEle; + } else { + size = tDataTypes[type].bytes * nEle; + } + + // alloc + code = tRealloc(ppOut, size); + if (code) goto _exit; + + // decode + if (cmprAlg == NO_COMPRESSION) { + ASSERT(szInput == size); + memcpy(*ppOut, pInput, size); + } else { + if (cmprAlg == TWO_STAGE_COMP) { + code = tRealloc(ppBuf, size + COMP_OVERFLOW_BYTES); + if (code) goto _exit; + + int32_t n = + tDataTypes[type].decompFunc(pInput, szInput, nEle, *ppOut, size, cmprAlg, *ppBuf, size + COMP_OVERFLOW_BYTES); + if (n <= 0) { + code = TSDB_CODE_COMPRESS_ERROR; + goto _exit; + } + } + } + +_exit: + return code; +} + static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { - TdFilePtr pFD = pBlock->last ? pReader->pLastFD : pReader->pDataFD; + TdFilePtr pFD = pReader->pDataFD; SSubBlock *pSubBlock = &pBlock->aSubBlock[iSubBlock]; SArray *aBlockCol = NULL; int32_t code = 0; @@ -974,7 +1002,7 @@ static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, S goto _err; } - code = tsdbReadBlockCol(pSubBlock, *ppBuf1, aBlockCol); + code = tsdbReadBlockCol(*ppBuf1, pSubBlock->szBlock, NULL /*todo*/, aBlockCol); if (code) goto _err; code = tsdbReadBlockDataKey(pBlockData, pSubBlock, *ppBuf1 + pSubBlock->szBlockCol + sizeof(TSCKSUM), ppBuf2); @@ -1093,13 +1121,13 @@ _err: return code; } -static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, - SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { +static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlock *pBlock, int32_t iSubBlock, SBlockData *pBlockData, + uint8_t **ppBuf1, uint8_t **ppBuf2) { int32_t code = 0; uint8_t *p; int64_t size; int64_t n; - TdFilePtr pFD = pBlock->last ? pReader->pLastFD : pReader->pDataFD; + TdFilePtr pFD = pReader->pDataFD; SSubBlock *pSubBlock = &pBlock->aSubBlock[iSubBlock]; SArray *aBlockCol = NULL; @@ -1175,20 +1203,18 @@ _err: return code; } -int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, - uint8_t **ppBuf1, uint8_t **ppBuf2) { - int32_t code = 0; - TdFilePtr pFD = pBlock->last ? pReader->pLastFD : pReader->pDataFD; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - int32_t iSubBlock; +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2) { + int32_t code = 0; + uint8_t *pBuf1 = NULL; + uint8_t *pBuf2 = NULL; if (!ppBuf1) ppBuf1 = &pBuf1; if (!ppBuf2) ppBuf2 = &pBuf2; // read the first sub-block - iSubBlock = 0; - code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData, ppBuf1, ppBuf2); + int32_t iSubBlock = 0; + code = tsdbReadSubBlockData(pReader, pBlock, iSubBlock, pBlockData, ppBuf1, ppBuf2); if (code) goto _err; // read remain block data and do merg @@ -1199,7 +1225,7 @@ int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *p tBlockDataInit(pBlockData1); tBlockDataInit(pBlockData2); for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); + code = tsdbReadSubBlockData(pReader, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); if (code) { tBlockDataClear(pBlockData1, 1); tBlockDataClear(pBlockData2, 1); @@ -1230,14 +1256,138 @@ int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *p ASSERT(tsdbKeyCmprFn(&pBlock->minKey, &TSDBROW_KEY(&tBlockDataFirstRow(pBlockData))) == 0); ASSERT(tsdbKeyCmprFn(&pBlock->maxKey, &TSDBROW_KEY(&tBlockDataLastRow(pBlockData))) == 0); - if (pBuf1) tFree(pBuf1); - if (pBuf2) tFree(pBuf2); + tFree(pBuf1); + tFree(pBuf2); return code; _err: tsdbError("vgId:%d, tsdb read block data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - if (pBuf1) tFree(pBuf1); - if (pBuf2) tFree(pBuf2); + tFree(pBuf1); + tFree(pBuf2); + return code; +} + +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, + uint8_t **ppBuf2) { + int32_t code = 0; + + tBlockDataReset(pBlockData); + + uint8_t *pBuf1 = NULL; + uint8_t *pBuf2 = NULL; + if (!ppBuf1) ppBuf1 = &pBuf1; + if (!ppBuf2) ppBuf2 = &pBuf2; + + // realloc + code = tRealloc(ppBuf1, pBlockL->szBlock); + if (code) goto _err; + + // seek + int64_t n = taosLSeekFile(pReader->pLastFD, pBlockL->offset, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // read + n = taosReadFile(pReader->pLastFD, *ppBuf1, pBlockL->szBlock); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n < pBlockL->szBlock) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // decode block col + SBlockDataHdr hdr; + SArray *aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); + uint8_t *p = *ppBuf1; + if (aBlockCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + code = tsdbReadBlockCol(p, pBlockL->szBlockCol, &hdr, aBlockCol); + if (code) goto _err; + p += pBlockL->szBlockCol + sizeof(TSCKSUM); + + // checksum + if (!taosCheckChecksumWhole(p, pBlockL->szUid + pBlockL->szVer + pBlockL->szTSKEY + sizeof(TSCKSUM))) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // UID + if (hdr.uid == 0) { + code = tsdbReadDataArray(p, pBlockL->szUid, pBlockL->nRow, TSDB_DATA_TYPE_BIGINT, pBlockL->cmprAlg, + (uint8_t **)&pBlockData->aUid, ppBuf2); + if (code) goto _err; + } else { + ASSERT(pBlockL->szUid == 0); + } + p += pBlockL->szUid; + + // VERSION + code = tsdbReadDataArray(p, pBlockL->szVer, pBlockL->nRow, TSDB_DATA_TYPE_BIGINT, pBlockL->cmprAlg, + (uint8_t **)&pBlockData->aVersion, ppBuf2); + if (code) goto _err; + p += pBlockL->szVer; + + // TSKEY + code = tsdbReadDataArray(p, pBlockL->szTSKEY, pBlockL->nRow, TSDB_DATA_TYPE_TIMESTAMP, pBlockL->cmprAlg, + (uint8_t **)&pBlockData->aTSKEY, ppBuf2); + if (code) goto _err; + p += pBlockL->szTSKEY; + p += sizeof(TSCKSUM); + + // COLUMN + code = tBlockDataSetSchema(pBlockData, NULL, hdr.suid, hdr.uid); + if (code) goto _err; + + for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(aBlockCol); iBlockCol++) { + SBlockCol *pBlockCol = (SBlockCol *)taosArrayGet(aBlockCol, iBlockCol); + SColData *pColData; + + // checksum + if (!taosCheckChecksumWhole(p, pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM))) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // add SColData + code = tBlockDataAddColData(pBlockData, iBlockCol, &pColData); + if (code) goto _err; + tColDataInit(pColData, pBlockCol->cid, pBlockCol->type, pBlockCol->smaOn); + pColData->nVal = pBlockL->nRow; + pColData->flag = pBlockCol->flag; + + // bitmap + if (pBlockCol->szBitmap) { + code = tsdbReadDataArray(p, pBlockCol->szBitmap, ); + if (code) goto _err; + } + p += pBlockCol->szBitmap; + + // offset + if (pBlockCol->szOffset) { + code = tsdbReadDataArray(p, pBlockCol->szOffset, ); + if (code) goto _err; + } + p += pBlockCol->szOffset; + + // value + pColData->nData = pBlockCol->szOrigin; + if (pColData->nData) { + // TODO + } + } + + taosArrayDestroy(aBlockCol); + + return code; + +_err: + tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } @@ -1832,10 +1982,12 @@ static int32_t tsdbWriteColData(SColData *pColData, SBlockCol *pBlockCol, int8_t // VALUE if (pColData->flag != (HAS_NULL | HAS_NONE)) { + ASSERT(pColData->nData); code = tsdbWriteDataArray(pColData->pData, pColData->nData, pColData->type, cmprAlg, &pBlockCol->szValue, ppBuf1, nBuf1 + n, ppBuf2); if (code) goto _err; } else { + ASSERT(pColData->nData == 0); pBlockCol->szValue = 0; } n += pBlockCol->szValue; @@ -2073,11 +2225,11 @@ int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock pBlockL->minUid = pBlockData->aUid[0]; pBlockL->maxUid = pBlockData->aUid[pBlockData->nRow - 1]; } + pBlockL->minVer = VERSION_MAX; + pBlockL->maxVer = VERSION_MIN; pBlockL->nRow = pBlockData->nRow; pBlockL->offset = pWriter->fLast.size; pBlockL->cmprAlg = cmprAlg; - pBlockL->minVer = VERSION_MAX; - pBlockL->maxVer = VERSION_MIN; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { pBlockL->minVer = TMIN(pBlockL->minVer, pBlockData->aVersion[iRow]); pBlockL->maxVer = TMAX(pBlockL->maxVer, pBlockData->aVersion[iRow]); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index c17a7f4a09..1ae241f0c1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -348,11 +348,11 @@ int32_t tPutBlockCol(uint8_t *p, void *ph) { n += tPutI8(p ? p + n : p, pBlockCol->flag); if (pBlockCol->flag != HAS_NULL) { - n += tPutI32v(p ? p + n : p, pBlockCol->offset); + n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin); n += tPutI32v(p ? p + n : p, pBlockCol->szBitmap); n += tPutI32v(p ? p + n : p, pBlockCol->szOffset); n += tPutI32v(p ? p + n : p, pBlockCol->szValue); - n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin); + n += tPutI32v(p ? p + n : p, pBlockCol->offset); } return n; @@ -370,11 +370,11 @@ int32_t tGetBlockCol(uint8_t *p, void *ph) { ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE)); if (pBlockCol->flag != HAS_NULL) { - n += tGetI32v(p + n, &pBlockCol->offset); + n += tGetI32v(p + n, &pBlockCol->szOrigin); n += tGetI32v(p + n, &pBlockCol->szBitmap); n += tGetI32v(p + n, &pBlockCol->szOffset); n += tGetI32v(p + n, &pBlockCol->szValue); - n += tGetI32v(p + n, &pBlockCol->szOrigin); + n += tGetI32v(p + n, &pBlockCol->offset); } return n; From 51538a1710ba247f0cfd29e64a463342164b5fe1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 5 Aug 2022 09:47:26 +0000 Subject: [PATCH 012/194] more work --- source/common/src/ttypes.c | 4 +- source/dnode/vnode/src/inc/tsdb.h | 76 ++++--- source/dnode/vnode/src/tsdb/tsdbRead.c | 24 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 10 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 208 +++++++++++++++++- 5 files changed, 271 insertions(+), 51 deletions(-) diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 156b66ae86..fee89e2f37 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -392,10 +392,10 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { getStatics_i64}, {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f}, {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d}, - {TSDB_DATA_TYPE_VARCHAR, 6, 0, "VARCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, + {TSDB_DATA_TYPE_VARCHAR, 6, 1, "VARCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64}, - {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_NCHAR, 5, 1, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_u8}, {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 32037cbf19..72296e4551 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -47,6 +47,7 @@ typedef struct SBlockL SBlockL; typedef struct SColData SColData; typedef struct SBlockDataHdr SBlockDataHdr; typedef struct SBlockData SBlockData; +typedef struct SDiskData SDiskData; typedef struct SDelFile SDelFile; typedef struct SHeadFile SHeadFile; typedef struct SDataFile SDataFile; @@ -149,6 +150,11 @@ SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); +// SDiskData +int32_t tDiskDataInit(SDiskData *pDiskData); +void tDiskDataClear(SDiskData *pDiskData); +int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); +int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); // SDelIdx int32_t tPutDelIdx(uint8_t *p, void *ph); int32_t tGetDelIdx(uint8_t *p, void *ph); @@ -392,38 +398,35 @@ struct SMapData { }; typedef struct { - int16_t cid; - int8_t type; - int8_t smaOn; - int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE - int32_t szOrigin; // original column value size (only save for variant data type) - int32_t szBitmap; // bitmap size - int32_t szOffset; // size of offset, only for variant-length data type - int32_t szValue; // compressed column value size - int32_t offset; + int16_t cid; + int8_t type; + int8_t smaOn; + int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE + int32_t szOrigin; // original column value size (only save for variant data type) + int32_t szBitmap; // bitmap size + int32_t szOffset; // size of offset, only for variant-length data type + int32_t szValue; // compressed column value size + int32_t offset; + uint8_t **ppData; } SBlockCol; typedef struct { - int32_t nRow; - int8_t cmprAlg; - int64_t offset; // block data offset - int32_t szBlockCol; // SBlockCol size - int32_t szVersion; // VERSION size - int32_t szTSKEY; // TSKEY size - int32_t szBlock; // total block size - int64_t sOffset; // sma offset - int32_t nSma; // sma size + int64_t offset; // block data offset + int32_t szBlock; + int32_t szKey; } SSubBlock; struct SBlock { TSDBKEY minKey; TSDBKEY maxKey; - int64_t minVersion; - int64_t maxVersion; + int64_t minVer; + int64_t maxVer; int32_t nRow; int8_t hasDup; int8_t nSubBlock; SSubBlock aSubBlock[TSDB_MAX_SUBBLOCKS]; + int64_t sOffset; // sma offset + int32_t nSma; // sma size }; struct SBlockL { @@ -433,13 +436,6 @@ struct SBlockL { int64_t minVer; int64_t maxVer; int32_t nRow; - int64_t offset; - int8_t cmprAlg; - int32_t szBlockCol; - int32_t szUid; - int32_t szVer; - int32_t szTSKEY; - int32_t szBlock; }; struct SColData { @@ -502,13 +498,17 @@ struct SDelIdx { int64_t size; }; -#pragma pack(push, 1) struct SBlockDataHdr { uint32_t delimiter; + int32_t nRow; int64_t suid; int64_t uid; + int32_t szUid; + int32_t szVer; + int32_t szKey; + int32_t szBlkCol; + int8_t cmprAlg; }; -#pragma pack(pop) struct SDelFile { volatile int32_t nRef; @@ -596,6 +596,24 @@ struct STsdbReadSnap { STsdbFS fs; }; +struct SDiskData { + int8_t cmprAlg; + int32_t nRow; + int64_t suid; + int64_t uid; + int32_t szUid; + int32_t szVer; + int32_t szKey; + + uint8_t *pUid; + uint8_t *pVer; + uint8_t *pKey; + SArray *aBlockCol; // SArray + int32_t nBuf; + SArray *aBuf; // SArray + uint8_t *pBuf; +}; + // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index cadb6fefd2..84ecaa605d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -630,7 +630,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_ } // 2. version range check - if (block.minVersion > pReader->verRange.maxVer || block.maxVersion < pReader->verRange.minVer) { + if (block.minVer > pReader->verRange.maxVer || block.maxVer < pReader->verRange.minVer) { continue; } @@ -766,7 +766,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, unDumpedRows, - pBlock->minVersion, pBlock->maxVersion, elapsedTime, pReader->idStr); + pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; } @@ -798,7 +798,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI tsdbDebug("%p load file block into buffer, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, - pBlock->minVersion, pBlock->maxVersion, elapsedTime, pReader->idStr); + pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -1011,8 +1011,8 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) { static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SBlock* pBlock) { return (pWindow->ekey < pBlock->maxKey.ts && pWindow->ekey >= pBlock->minKey.ts) || (pWindow->skey > pBlock->minKey.ts && pWindow->skey <= pBlock->maxKey.ts) || - (pVerRange->minVer > pBlock->minVersion && pVerRange->minVer <= pBlock->maxVersion) || - (pVerRange->maxVer < pBlock->maxVersion && pVerRange->maxVer >= pBlock->minVersion); + (pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) || + (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); } static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, @@ -1092,8 +1092,8 @@ static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SBlock* pBlock) } static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVerRange) { - return (key.ts >= pBlock->minKey.ts && key.ts <= pBlock->maxKey.ts) && (pBlock->maxVersion >= pVerRange->minVer) && - (pBlock->minVersion <= pVerRange->maxVer); + return (key.ts >= pBlock->minKey.ts && key.ts <= pBlock->maxKey.ts) && (pBlock->maxVer >= pVerRange->minVer) && + (pBlock->minVer <= pVerRange->maxVer); } static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) { @@ -1102,11 +1102,11 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) { TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); if (p->ts >= pBlock->minKey.ts && p->ts <= pBlock->maxKey.ts) { - if (p->version >= pBlock->minVersion) { + if (p->version >= pBlock->minVer) { return true; } } else if (p->ts < pBlock->minKey.ts) { // p->ts < pBlock->minKey.ts - if (p->version >= pBlock->minVersion) { + if (p->version >= pBlock->minVer) { if (i < num - 1) { TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1); if (i + 1 == num - 1) { // pnext is the last point @@ -1114,7 +1114,7 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons return true; } } else { - if (pnext->ts >= pBlock->minKey.ts && pnext->version >= pBlock->minVersion) { + if (pnext->ts >= pBlock->minKey.ts && pnext->version >= pBlock->minVer) { return true; } } @@ -2512,9 +2512,7 @@ void* tsdbGetIvtIdx(SMeta* pMeta) { return metaGetIvtIdx(pMeta); } -uint64_t getReaderMaxVersion(STsdbReader *pReader) { - return pReader->verRange.maxVer; -} +uint64_t getReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; } /** * @brief Get all suids since suid diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index bc4656278c..eeb11abc97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1909,8 +1909,8 @@ static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { pBlock->maxKey = key; } - pBlock->minVersion = TMIN(pBlock->minVersion, key.version); - pBlock->maxVersion = TMAX(pBlock->maxVersion, key.version); + pBlock->minVer = TMIN(pBlock->minVer, key.version); + pBlock->maxVer = TMAX(pBlock->maxVer, key.version); } pBlock->nRow += pBlockData->nRow; } @@ -2310,9 +2310,13 @@ int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock pBlockL->szBlock = pBlockL->szBlockCol + sizeof(TSCKSUM) + nBuf1; pWriter->fLast.size += pBlockL->szBlock; + tFree(pBuf1); + tFree(pBuf2); return code; _err: + tFree(pBuf1); + tFree(pBuf2); return code; } @@ -2426,4 +2430,4 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { _err: tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 1ae241f0c1..13d293f86b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -227,8 +227,8 @@ int32_t tPutBlock(uint8_t *p, void *ph) { n += tPutTSDBKEY(p ? p + n : p, &pBlock->minKey); n += tPutTSDBKEY(p ? p + n : p, &pBlock->maxKey); - n += tPutI64v(p ? p + n : p, pBlock->minVersion); - n += tPutI64v(p ? p + n : p, pBlock->maxVersion); + n += tPutI64v(p ? p + n : p, pBlock->minVer); + n += tPutI64v(p ? p + n : p, pBlock->maxVer); n += tPutI32v(p ? p + n : p, pBlock->nRow); n += tPutI8(p ? p + n : p, pBlock->hasDup); n += tPutI8(p ? p + n : p, pBlock->nSubBlock); @@ -253,8 +253,8 @@ int32_t tGetBlock(uint8_t *p, void *ph) { n += tGetTSDBKEY(p + n, &pBlock->minKey); n += tGetTSDBKEY(p + n, &pBlock->maxKey); - n += tGetI64v(p + n, &pBlock->minVersion); - n += tGetI64v(p + n, &pBlock->maxVersion); + n += tGetI64v(p + n, &pBlock->minVer); + n += tGetI64v(p + n, &pBlock->maxVer); n += tGetI32v(p + n, &pBlock->nRow); n += tGetI8(p + n, &pBlock->hasDup); n += tGetI8(p + n, &pBlock->nSubBlock); @@ -1509,6 +1509,206 @@ int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData) { return n; } +// SDiskData ============================== +static int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, + int32_t *szOut, uint8_t **ppBuf) { + int32_t code = 0; + + ASSERT(szIn > 0 && ppOut); + + if (cmprAlg == NO_COMPRESSION) { + code = tRealloc(ppOut, nOut + szIn); + if (code) goto _exit; + + memcpy(*ppOut + nOut, pIn, szIn); + *szOut = szIn; + } else { + int32_t size = szIn + COMP_OVERFLOW_BYTES; + + code = tRealloc(ppOut, nOut + size); + if (code) goto _exit; + + if (cmprAlg == TWO_STAGE_COMP) { + ASSERT(ppBuf); + code = tRealloc(ppBuf, size); + if (code) goto _exit; + } + + *szOut = + tDataTypes[type].compFunc(pIn, szIn, szIn / tDataTypes[type].bytes, *ppOut + nOut, size, cmprAlg, *ppBuf, size); + if (*szOut <= 0) { + code = TSDB_CODE_COMPRESS_ERROR; + goto _exit; + } + } + +_exit: + return code; +} + +static int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf) { + int32_t code = 0; + + int32_t n = 0; + // bitmap + if (pColData->flag != HAS_VALUE) { + code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, + pBlockCol->ppData, n, &pBlockCol->szBitmap, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szBitmap = 0; + } + n += pBlockCol->szBitmap; + + // offset + if (IS_VAR_DATA_TYPE(pColData->type)) { + code = tsdbCmprData((uint8_t *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, + pBlockCol->ppData, n, &pBlockCol->szOffset, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szOffset = 0; + } + n += pBlockCol->szOffset; + + // value + if (pColData->flag != (HAS_NULL | HAS_NONE)) { + code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, pBlockCol->ppData, n, + &pBlockCol->szValue, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szValue = 0; + } + n += pBlockCol->szValue; + + // checksum + n += sizeof(TSCKSUM); + taosCalcChecksumAppend(0, *ppBuf, n); + +_exit: + return code; +} + +static int32_t tsdbDecmprData() { + int32_t code = 0; + // TODO + return code; +} + +int32_t tDiskDataInit(SDiskData *pDiskData) { + int32_t code = 0; + + pDiskData->aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); + if (pDiskData->aBlockCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pDiskData->aBuf = taosArrayInit(0, sizeof(uint8_t *)); + if (pDiskData->aBuf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + +_exit: + return code; +} + +void tDiskDataClear(SDiskData *pDiskData) { + taosArrayDestroy(pDiskData->aBlockCol); + for (int32_t i = 0; i < taosArrayGetSize(pDiskData->aBuf); i++) { + tFree((uint8_t *)taosArrayGet(pDiskData->aBuf, i)); + } + taosArrayDestroy(pDiskData->aBuf); +} + +static uint8_t **tDiskDataAllocBuf(SDiskData *pDiskData) { + if (pDiskData->nBuf >= taosArrayGetSize(pDiskData->aBuf)) { + uint8_t *p = NULL; + if (taosArrayPush(pDiskData->aBuf, &p) == NULL) { + return NULL; + } + } + + ASSERT(pDiskData->nBuf < taosArrayGetSize(pDiskData->aBuf)); + uint8_t **pp = taosArrayGet(pDiskData->aBuf, pDiskData->nBuf); + pDiskData->nBuf++; + return pp; +} + +int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg) { + int32_t code = 0; + + ASSERT(pBlockData->nRow > 0); + + pDiskData->cmprAlg = cmprAlg; + pDiskData->nRow = pBlockData->nRow; + pDiskData->suid = pBlockData->suid; + pDiskData->uid = pBlockData->uid; + pDiskData->szUid = 0; + pDiskData->szVer = 0; + pDiskData->szKey = 0; + taosArrayClear(pDiskData->aBlockCol); + pDiskData->nBuf = 0; + + // uid + if (pDiskData->uid == 0) { + code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pDiskData->pUid, &pDiskData->szUid, &pDiskData->pBuf); + if (code) goto _exit; + } + + // version + code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, &pDiskData->pVer, &pDiskData->szVer, &pDiskData->pBuf); + if (code) goto _exit; + + // tskey + code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, + cmprAlg, &pDiskData->pKey, &pDiskData->szKey, &pDiskData->pBuf); + if (code) goto _exit; + + // columns + int32_t offset = 0; + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + + if (pColData->flag == HAS_NONE) continue; + + SBlockCol blockCol = {.cid = pColData->cid, + .type = pColData->type, + .smaOn = pColData->smaOn, + .flag = pColData->flag, + .szOrigin = pColData->nData}; + + if (pColData->flag != HAS_NULL) { + // alloc a buffer + blockCol.ppData = tDiskDataAllocBuf(pDiskData); + if (blockCol.ppData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + // compress + code = tsdbCmprColData(pColData, cmprAlg, &blockCol); + if (code) goto _exit; + + // update offset + blockCol.offset = offset; + offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; + } + + if (taosArrayPush(pDiskData->aBlockCol, &blockCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + } + +_exit: + return code; +} +int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); +int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); + // ALGORITHM ============================== void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { SColVal colVal; From 16f556354a979c0b0cb6664c800e5a69fdc6a3e7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 5 Aug 2022 15:22:05 +0000 Subject: [PATCH 013/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 88 ++- source/dnode/vnode/src/tsdb/tsdbCommit.c | 84 ++- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 518 +++++------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 183 +++++-- 4 files changed, 359 insertions(+), 514 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 72296e4551..08331b1327 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -45,7 +45,7 @@ typedef struct SBlockIdx SBlockIdx; typedef struct SBlock SBlock; typedef struct SBlockL SBlockL; typedef struct SColData SColData; -typedef struct SBlockDataHdr SBlockDataHdr; +typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SBlockData SBlockData; typedef struct SDiskData SDiskData; typedef struct SDelFile SDelFile; @@ -155,6 +155,9 @@ int32_t tDiskDataInit(SDiskData *pDiskData); void tDiskDataClear(SDiskData *pDiskData); int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); +// SDiskDataHdr +int32_t tPutDiskDataHdr(uint8_t *p, void *ph); +int32_t tGetDiskDataHdr(uint8_t *p, void *ph); // SDelIdx int32_t tPutDelIdx(uint8_t *p, void *ph); int32_t tGetDelIdx(uint8_t *p, void *ph); @@ -236,10 +239,8 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBuf, SBlockIdx *pBlockIdx); int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf); -int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, - uint8_t **ppBuf2, int8_t cmprAlg); -int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, - uint8_t **ppBuf2, int8_t cmprAlg); +int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, + int8_t cmprAlg, int8_t toLast, uint8_t **ppBuf); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader @@ -403,9 +404,9 @@ typedef struct { int8_t smaOn; int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE int32_t szOrigin; // original column value size (only save for variant data type) - int32_t szBitmap; // bitmap size - int32_t szOffset; // size of offset, only for variant-length data type - int32_t szValue; // compressed column value size + int32_t szBitmap; // bitmap size, 0 only for flag == HAS_VAL + int32_t szOffset; // offset size, 0 only for non-variant-length type + int32_t szValue; // value size, 0 when flag == (HAS_NULL | HAS_NONE) int32_t offset; uint8_t **ppData; } SBlockCol; @@ -414,28 +415,33 @@ typedef struct { int64_t offset; // block data offset int32_t szBlock; int32_t szKey; -} SSubBlock; +} SBlockInfo; + +typedef struct { + int64_t offset; + int32_t size; +} SSmaInfo; struct SBlock { - TSDBKEY minKey; - TSDBKEY maxKey; - int64_t minVer; - int64_t maxVer; - int32_t nRow; - int8_t hasDup; - int8_t nSubBlock; - SSubBlock aSubBlock[TSDB_MAX_SUBBLOCKS]; - int64_t sOffset; // sma offset - int32_t nSma; // sma size + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t minVer; + int64_t maxVer; + int32_t nRow; + int8_t hasDup; + int8_t nSubBlock; + SBlockInfo aSubBlock[TSDB_MAX_SUBBLOCKS]; + SSmaInfo smaInfo; }; struct SBlockL { - int64_t suid; - int64_t minUid; - int64_t maxUid; - int64_t minVer; - int64_t maxVer; - int32_t nRow; + int64_t suid; + int64_t minUid; + int64_t maxUid; + int64_t minVer; + int64_t maxVer; + int32_t nRow; + SBlockInfo bInfo; }; struct SColData { @@ -498,15 +504,15 @@ struct SDelIdx { int64_t size; }; -struct SBlockDataHdr { +struct SDiskDataHdr { uint32_t delimiter; - int32_t nRow; int64_t suid; int64_t uid; int32_t szUid; int32_t szVer; int32_t szKey; int32_t szBlkCol; + int32_t nRow; int8_t cmprAlg; }; @@ -575,6 +581,14 @@ struct SDelFWriter { TdFilePtr pWriteH; }; +struct SDiskData { + SDiskDataHdr hdr; + uint8_t **ppKey; + SArray *aBlockCol; // SArray + int32_t nBuf; + SArray *aBuf; // SArray +}; + struct SDataFWriter { STsdb *pTsdb; SDFileSet wSet; @@ -588,6 +602,8 @@ struct SDataFWriter { SDataFile fData; SLastFile fLast; SSmaFile fSma; + + SDiskData dData; }; struct STsdbReadSnap { @@ -596,24 +612,6 @@ struct STsdbReadSnap { STsdbFS fs; }; -struct SDiskData { - int8_t cmprAlg; - int32_t nRow; - int64_t suid; - int64_t uid; - int32_t szUid; - int32_t szVer; - int32_t szKey; - - uint8_t *pUid; - uint8_t *pVer; - uint8_t *pKey; - SArray *aBlockCol; // SArray - int32_t nBuf; - SArray *aBuf; // SArray - uint8_t *pBuf; -}; - // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index e5c6ee44a0..cd94762c7b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -494,10 +494,11 @@ _exit: } static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { - int32_t code = 0; - SBlock block; + int32_t code = 0; + SBlockData *pBlockData = &pCommitter->dWriter.bData; + SBlock block; - ASSERT(pCommitter->dWriter.bData.nRow > 0); + ASSERT(pBlockData->nRow > 0); if (pBlock) { block = *pBlock; // as a subblock @@ -505,37 +506,84 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { tBlockReset(&block); // as a new block } - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bData, &block, NULL, NULL, - pCommitter->cmprAlg); - if (code) goto _exit; + // statistic + block.nRow += pBlockData->nRow; + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; + if (iRow == 0) { + if (tsdbKeyCmprFn(&block.minKey, &key) > 0) { + block.minKey = key; + } + } else { + if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) { + block.hasDup = 1; + } + } + + if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&block.maxKey, &key) < 0) { + block.maxKey = key; + } + + block.minVer = TMIN(block.minVer, key.version); + block.maxVer = TMAX(block.maxVer, key.version); + } + + // write + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock++], + &block.smaInfo, pCommitter->cmprAlg, 0, NULL); + if (code) goto _err; + + // put SBlock code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); - if (code) goto _exit; + if (code) goto _err; - tBlockDataClearData(&pCommitter->dWriter.bData); + // clear + tBlockDataClearData(pBlockData); -_exit: + return code; + +_err: + tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { - int32_t code = 0; - SBlockL blockL; + int32_t code = 0; + SBlockL blockL; + SBlockData *pBlockData = &pCommitter->dWriter.bDatal; - ASSERT(pCommitter->dWriter.bDatal.nRow > 0); + ASSERT(pBlockData->nRow > 0); - code = tsdbWriteLastBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, &blockL, NULL, NULL, - pCommitter->cmprAlg); - if (code) goto _exit; + // statistic + blockL.suid = pBlockData->suid; + blockL.nRow = pBlockData->nRow; + blockL.minVer = VERSION_MAX; + blockL.maxVer = VERSION_MIN; + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + blockL.minVer = TMIN(blockL.minVer, pBlockData->aVersion[iRow]); + blockL.maxVer = TMIN(blockL.maxVer, pBlockData->aVersion[iRow]); + } + blockL.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0]; + blockL.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; + // write + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1, NULL); + if (code) goto _err; + + // push SBlockL if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + goto _err; } - tBlockDataClearData(&pCommitter->dWriter.bDatal); + // clear + tBlockDataClearData(pBlockData); -_exit: + return code; + +_err: + tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index eeb11abc97..c542ab5583 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -85,11 +85,11 @@ _err: } int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf, SDelIdx *pDelIdx) { - int32_t code = 0; - uint8_t *pBuf = NULL; - int64_t size; - int64_t n; - SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pDelIdx->suid, .uid = pDelIdx->uid}; + int32_t code = 0; + uint8_t *pBuf = NULL; + int64_t size; + int64_t n; + SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pDelIdx->suid, .uid = pDelIdx->uid}; if (!ppBuf) ppBuf = &pBuf; @@ -106,7 +106,7 @@ int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf // build n = 0; - *(SBlockDataHdr *)(*ppBuf) = hdr; + *(SDiskDataHdr *)(*ppBuf) = hdr; n += sizeof(hdr); for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { n += tPutDelData(*ppBuf + n, taosArrayGet(aDelData, iDelData)); @@ -306,13 +306,13 @@ _exit: } int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf) { - int32_t code = 0; - int64_t offset = pDelIdx->offset; - int64_t size = pDelIdx->size; - int64_t n; - uint8_t *pBuf = NULL; - SBlockDataHdr *pHdr; - SDelData *pDelData = &(SDelData){0}; + int32_t code = 0; + int64_t offset = pDelIdx->offset; + int64_t size = pDelIdx->size; + int64_t n; + uint8_t *pBuf = NULL; + SDiskDataHdr *pHdr; + SDelData *pDelData = &(SDelData){0}; if (!ppBuf) ppBuf = &pBuf; @@ -344,7 +344,7 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData // // decode n = 0; - pHdr = (SBlockDataHdr *)(*ppBuf + n); + pHdr = (SDiskDataHdr *)(*ppBuf + n); ASSERT(pHdr->delimiter == TSDB_FILE_DLMT); ASSERT(pHdr->suid == pDelIdx->suid); ASSERT(pHdr->uid == pDelIdx->uid); @@ -669,13 +669,13 @@ _err: } int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock, uint8_t **ppBuf) { - int32_t code = 0; - int64_t offset = pBlockIdx->offset; - int64_t size = pBlockIdx->size; - uint8_t *pBuf = NULL; - int64_t n; - int64_t tn; - SBlockDataHdr hdr; + int32_t code = 0; + int64_t offset = pBlockIdx->offset; + int64_t size = pBlockIdx->size; + uint8_t *pBuf = NULL; + int64_t n; + int64_t tn; + SDiskDataHdr hdr; if (!ppBuf) ppBuf = &pBuf; @@ -706,7 +706,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl } // decode - hdr = *(SBlockDataHdr *)(*ppBuf); + hdr = *(SDiskDataHdr *)(*ppBuf); ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(hdr.suid == pBlockIdx->suid); ASSERT(hdr.uid == pBlockIdx->uid); @@ -729,7 +729,7 @@ _err: return code; } -static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SSubBlock *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { +static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; int64_t size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); int64_t n; @@ -784,7 +784,7 @@ _err: return code; } -static int32_t tsdbReadColDataImpl(SSubBlock *pSubBlock, SBlockCol *pBlockCol, SColData *pColData, uint8_t *pBuf, +static int32_t tsdbReadColDataImpl(SBlockInfo *pSubBlock, SBlockCol *pBlockCol, SColData *pColData, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; int64_t size; @@ -877,7 +877,7 @@ _err: return code; } -static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SBlockDataHdr *pHdr, SArray *aBlockCol) { +static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SDiskDataHdr *pHdr, SArray *aBlockCol) { int32_t code = 0; int32_t n = 0; SBlockCol blockCol; @@ -890,8 +890,8 @@ static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SBlockDataHdr } // hdr - *pHdr = *(SBlockDataHdr *)pBuf; - n += sizeof(SBlockDataHdr); + *pHdr = *(SDiskDataHdr *)pBuf; + n += sizeof(SDiskDataHdr); // aBlockCol while (n < szBlockCol) { @@ -952,13 +952,13 @@ _exit: static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { - TdFilePtr pFD = pReader->pDataFD; - SSubBlock *pSubBlock = &pBlock->aSubBlock[iSubBlock]; - SArray *aBlockCol = NULL; - int32_t code = 0; - int64_t offset; - int64_t size; - int64_t n; + TdFilePtr pFD = pReader->pDataFD; + SBlockInfo *pSubBlock = &pBlock->aSubBlock[iSubBlock]; + SArray *aBlockCol = NULL; + int32_t code = 0; + int64_t offset; + int64_t size; + int64_t n; tBlockDataReset(pBlockData); pBlockData->nRow = pSubBlock->nRow; @@ -1123,13 +1123,13 @@ _err: static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlock *pBlock, int32_t iSubBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { - int32_t code = 0; - uint8_t *p; - int64_t size; - int64_t n; - TdFilePtr pFD = pReader->pDataFD; - SSubBlock *pSubBlock = &pBlock->aSubBlock[iSubBlock]; - SArray *aBlockCol = NULL; + int32_t code = 0; + uint8_t *p; + int64_t size; + int64_t n; + TdFilePtr pFD = pReader->pDataFD; + SBlockInfo *pSubBlock = &pBlock->aSubBlock[iSubBlock]; + SArray *aBlockCol = NULL; tBlockDataReset(pBlockData); @@ -1300,9 +1300,9 @@ int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *p } // decode block col - SBlockDataHdr hdr; - SArray *aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); - uint8_t *p = *ppBuf1; + SDiskDataHdr hdr; + SArray *aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); + uint8_t *p = *ppBuf1; if (aBlockCol == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1461,6 +1461,8 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + code = tDiskDataInit(&pWriter->dData); + if (code) goto _err; pWriter->pTsdb = pTsdb; pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, .fid = pSet->fid, @@ -1639,6 +1641,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { goto _err; } + tDiskDataClear(&(*ppWriter)->dData); taosMemoryFree(*ppWriter); _exit: *ppWriter = NULL; @@ -1786,17 +1789,17 @@ _err: } int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, uint8_t **ppBuf, SBlockIdx *pBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; - uint8_t *pBuf = NULL; - int64_t size; - int64_t n; + int32_t code = 0; + SHeadFile *pHeadFile = &pWriter->fHead; + SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; + uint8_t *pBuf = NULL; + int64_t size; + int64_t n; ASSERT(mBlock->nItem > 0); // prepare - size = sizeof(SBlockDataHdr) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); + size = sizeof(SDiskDataHdr) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); // alloc if (!ppBuf) ppBuf = &pBuf; @@ -1805,7 +1808,7 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, uint8_t **ppBuf, // build n = 0; - *(SBlockDataHdr *)(*ppBuf) = hdr; + *(SDiskDataHdr *)(*ppBuf) = hdr; n += sizeof(hdr); n += tPutMapData(*ppBuf + n, mBlock); taosCalcChecksumAppend(0, *ppBuf, size); @@ -1915,141 +1918,7 @@ static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { pBlock->nRow += pBlockData->nRow; } -static int32_t tsdbWriteDataArray(uint8_t *aData, int32_t nEle, int8_t type, int8_t cmprAlg, int32_t *rSize, - uint8_t **ppBuf1, int64_t nBuf1, uint8_t **ppBuf2) { - int32_t code = 0; - int32_t size; - - if (IS_VAR_DATA_TYPE(type)) { - size = nEle; - } else { - size = tDataTypes[type].bytes * nEle; - } - - if (cmprAlg == NO_COMPRESSION) { - code = tRealloc(ppBuf1, nBuf1 + size); - if (code) goto _exit; - - memcpy(*ppBuf1 + nBuf1, aData, size); - *rSize = size; - } else { - code = tRealloc(ppBuf1, size + COMP_OVERFLOW_BYTES); - if (code) goto _exit; - - if (cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf2, size + COMP_OVERFLOW_BYTES); - if (code) goto _exit; - } - - int32_t n = tDataTypes[type].compFunc(aData, tDataTypes[type].bytes * nEle, nEle, *ppBuf1 + nBuf1, - size + COMP_OVERFLOW_BYTES, cmprAlg, *ppBuf2, size + COMP_OVERFLOW_BYTES); - if (n <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _exit; - } - *rSize = n; - } - -_exit: - return code; -} - -static int32_t tsdbWriteColData(SColData *pColData, SBlockCol *pBlockCol, int8_t cmprAlg, uint8_t **ppBuf1, - int64_t nBuf1, uint8_t **ppBuf2) { - int32_t code = 0; - int64_t size; - int64_t n = 0; - - // BITMAP - if (pColData->flag != HAS_VALUE) { - code = tsdbWriteDataArray(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, - &pBlockCol->szBitmap, ppBuf1, nBuf1 + n, ppBuf2); - if (code) goto _err; - } else { - pBlockCol->szBitmap = 0; - } - n += pBlockCol->szBitmap; - - // OFFSET - if (IS_VAR_DATA_TYPE(pColData->type)) { - code = tsdbWriteDataArray((uint8_t *)pColData->aOffset, pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, - &pBlockCol->szOffset, ppBuf1, nBuf1 + n, ppBuf2); - if (code) goto _err; - } else { - pBlockCol->szOffset = 0; - } - n += pBlockCol->szOffset; - - // VALUE - if (pColData->flag != (HAS_NULL | HAS_NONE)) { - ASSERT(pColData->nData); - code = tsdbWriteDataArray(pColData->pData, pColData->nData, pColData->type, cmprAlg, &pBlockCol->szValue, ppBuf1, - nBuf1 + n, ppBuf2); - if (code) goto _err; - } else { - ASSERT(pColData->nData == 0); - pBlockCol->szValue = 0; - } - n += pBlockCol->szValue; - - // checksum - n += sizeof(TSCKSUM); - code = tRealloc(ppBuf1, nBuf1 + n); - if (code) goto _err; - taosCalcChecksumAppend(0, *ppBuf1 + nBuf1, n); - - return code; - -_err: - return code; -} - -static int32_t tsdbWriteBlockDataImpl(TdFilePtr pFD, SBlockDataHdr hdr, SArray *aBlockCol, uint8_t *pData, - int64_t nData, uint8_t **ppBuf, int32_t *szBlockCol) { - int32_t code = 0; - int32_t nBlockCol = taosArrayGetSize(aBlockCol); - int64_t size; - int64_t n; - - // HDR + SArray - *szBlockCol = sizeof(hdr); - for (int32_t iBlockCol = 0; iBlockCol < nBlockCol; iBlockCol++) { - (*szBlockCol) += tPutBlockCol(NULL, taosArrayGet(aBlockCol, iBlockCol)); - } - - code = tRealloc(ppBuf, *szBlockCol + sizeof(TSCKSUM)); - if (code) goto _err; - - n = 0; - memcpy(*ppBuf, &hdr, sizeof(hdr)); - n += sizeof(hdr); - for (int32_t iBlockCol = 0; iBlockCol < nBlockCol; iBlockCol++) { - n += tPutBlockCol(*ppBuf + n, taosArrayGet(aBlockCol, iBlockCol)); - } - taosCalcChecksumAppend(0, *ppBuf, *szBlockCol + sizeof(TSCKSUM)); - - ASSERT(n == *szBlockCol); - - n = taosWriteFile(pFD, *ppBuf, *szBlockCol + sizeof(TSCKSUM)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // SBlockData - n = taosWriteFile(pFD, pData, nData); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - return code; - -_err: - return code; -} - -static int32_t tsdbWriteBlockSma(TdFilePtr pFD, SBlockData *pBlockData, SSubBlock *pSubBlock, uint8_t **ppBuf) { +static int32_t tsdbWriteBlockSma(TdFilePtr pFD, SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t **ppBuf) { int32_t code = 0; int64_t n; SColData *pColData; @@ -2093,230 +1962,93 @@ _err: return code; } -int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock *pBlock, uint8_t **ppBuf1, - uint8_t **ppBuf2, int8_t cmprAlg) { - int32_t code = 0; +int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, + int8_t cmprAlg, int8_t toLast, uint8_t **ppBuf) { + int32_t code = 0; + TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; + SDiskData *pDiskData = &pWriter->dData; + uint8_t *pBuf = NULL; - ASSERT((pBlockData->suid && pBlockData->uid) || (!pBlockData->suid && pBlockData->uid)); + if (!ppBuf) ppBuf = &pBuf; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - - if (!ppBuf1) ppBuf1 = &pBuf1; - if (!ppBuf2) ppBuf2 = &pBuf2; - - tsdbUpdateBlockInfo(pBlockData, pBlock); - - SSubBlock *pSubBlock = &pBlock->aSubBlock[pBlock->nSubBlock++]; - - pSubBlock->nRow = pBlockData->nRow; - pSubBlock->cmprAlg = cmprAlg; - pSubBlock->offset = pWriter->fData.size; - - // ======================= BLOCK DATA ======================= - int64_t nBuf1 = 0; - - // VERSION - code = tsdbWriteDataArray((uint8_t *)pBlockData->aVersion, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pSubBlock->szVersion, ppBuf1, nBuf1, ppBuf2); + // convert + code = tBlockToDiskData(pBlockData, pDiskData, cmprAlg); if (code) goto _err; - nBuf1 += pSubBlock->szVersion; - // TSKEY - code = tsdbWriteDataArray((uint8_t *)pBlockData->aTSKEY, pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg, - &pSubBlock->szTSKEY, ppBuf1, nBuf1, ppBuf2); + // write the block + if (toLast) { + pBlkInfo->offset = pWriter->fLast.size; + } else { + pBlkInfo->offset = pWriter->fData.size; + } + + // HDR and KEY + int32_t size = tPutDiskDataHdr(NULL, &pDiskData->hdr); + code = tRealloc(ppBuf, size); if (code) goto _err; - nBuf1 += pSubBlock->szTSKEY; - // checksum - nBuf1 += sizeof(TSCKSUM); - code = tRealloc(ppBuf1, nBuf1); - if (code) goto _err; - taosCalcChecksumAppend(0, *ppBuf1, nBuf1); + tPutDiskDataHdr(*ppBuf, &pDiskData->hdr); - // COLUMNS - SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + TSCKSUM cksm = taosCalcChecksum(0, *ppBuf, size); + + int64_t n = taosWriteFile(pFD, *ppBuf, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); goto _err; } - int32_t offset = 0; - for (int32_t iCol = 0; iCol < taosArrayGetSize(pBlockData->aIdx); iCol++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); - SBlockCol blockCol = {0}; - ASSERT(pColData->flag); - - if (pColData->flag == HAS_NONE) continue; - - blockCol.cid = pColData->cid; - blockCol.type = pColData->type; - blockCol.smaOn = pColData->smaOn; - blockCol.flag = pColData->flag; - blockCol.szOrigin = pColData->nData; - - if (pColData->flag != HAS_NULL) { - code = tsdbWriteColData(pColData, &blockCol, cmprAlg, ppBuf1, nBuf1, ppBuf2); - if (code) goto _err; - - blockCol.offset = offset; - offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); - nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); - } - - if (taosArrayPush(aBlockCol, &blockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + cksm = taosCalcChecksum(cksm, *pDiskData->ppKey, pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey); + n = taosWriteFile(pFD, *pDiskData->ppKey, pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; } - // write - SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockData->suid, .uid = pBlockData->uid}; - code = tsdbWriteBlockDataImpl(pWriter->pDataFD, hdr, aBlockCol, *ppBuf1, nBuf1, ppBuf2, &pSubBlock->szBlockCol); - if (code) goto _err; - - pSubBlock->szBlock = pSubBlock->szBlockCol + sizeof(TSCKSUM) + nBuf1; - pWriter->fData.size += pSubBlock->szBlock; - - // ======================= BLOCK SMA ======================= - pSubBlock->sOffset = 0; - pSubBlock->nSma = 0; - - if (pBlock->nSubBlock > 1 || pBlock->hasDup) goto _exit; - - code = tsdbWriteBlockSma(pWriter->pSmaFD, pBlockData, pSubBlock, ppBuf1); - if (code) goto _err; - - if (pSubBlock->nSma > 0) { - pSubBlock->sOffset = pWriter->fSma.size; - pWriter->fSma.size += (sizeof(SColumnDataAgg) * pSubBlock->nSma + sizeof(TSCKSUM)); + n = taosWriteFile(pFD, &cksm, sizeof(cksm)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; } + pBlkInfo->szKey = size + pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey + sizeof(TSCKSUM); + + // SBlockCol + if (pDiskData->hdr.szBlkCol == 0) { + pBlkInfo->szBlock = pBlkInfo->szKey; + goto _write_sma; + } + + code = tRealloc(ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); + if (code) goto _err; + + n = 0; + for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { + n += tPutBlockCol(*ppBuf + n, taosArrayGet(pDiskData->aBlockCol, iBlockCol)); + } + taosCalcChecksumAppend(0, *ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); + + n = taosWriteFile(pFD, *ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { + } + +// ================= SMA ==================== +_write_sma: + if (toLast) goto _exit; + if (pSmaInfo == NULL) goto _exit; + + // TODO + _exit: - tFree(pBuf1); - tFree(pBuf2); - taosArrayDestroy(aBlockCol); + tFree(pBuf); return code; _err: - tsdbError("vgId:%d, write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf1); - tFree(pBuf2); - taosArrayDestroy(aBlockCol); - return code; -} - -int32_t tsdbWriteLastBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockL *pBlockL, uint8_t **ppBuf1, - uint8_t **ppBuf2, int8_t cmprAlg) { - int32_t code = 0; - - ASSERT((pBlockData->suid && !pBlockData->uid) || (!pBlockData->suid && pBlockData->uid)); - - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - - if (!ppBuf1) ppBuf1 = &pBuf1; - if (!ppBuf2) ppBuf2 = &pBuf2; - - pBlockL->suid = pBlockData->suid; - if (pBlockData->uid) { - pBlockL->maxUid = pBlockL->minUid = pBlockData->uid; - } else { - pBlockL->minUid = pBlockData->aUid[0]; - pBlockL->maxUid = pBlockData->aUid[pBlockData->nRow - 1]; - } - pBlockL->minVer = VERSION_MAX; - pBlockL->maxVer = VERSION_MIN; - pBlockL->nRow = pBlockData->nRow; - pBlockL->offset = pWriter->fLast.size; - pBlockL->cmprAlg = cmprAlg; - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - pBlockL->minVer = TMIN(pBlockL->minVer, pBlockData->aVersion[iRow]); - pBlockL->maxVer = TMAX(pBlockL->maxVer, pBlockData->aVersion[iRow]); - } - - // ======================= BLOCK DATA ======================= - int64_t nBuf1 = 0; - - // UID - if (pBlockData->uid == 0) { - code = tsdbWriteDataArray((uint8_t *)pBlockData->aUid, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pBlockL->szUid, ppBuf1, nBuf1, ppBuf2); - if (code) goto _err; - } else { - pBlockL->szUid = 0; - } - nBuf1 += pBlockL->szUid; - - // VERSION - code = tsdbWriteDataArray((uint8_t *)pBlockData->aVersion, pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pBlockL->szVer, ppBuf1, nBuf1, ppBuf2); - if (code) goto _err; - nBuf1 += pBlockL->szVer; - - // TSKEY - code = tsdbWriteDataArray((uint8_t *)pBlockData->aTSKEY, pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg, - &pBlockL->szTSKEY, ppBuf1, nBuf1, ppBuf2); - if (code) goto _err; - nBuf1 += pBlockL->szTSKEY; - - // checksum - nBuf1 += sizeof(TSCKSUM); - code = tRealloc(ppBuf1, nBuf1); - if (code) goto _err; - taosCalcChecksumAppend(0, *ppBuf1, nBuf1); - - // COLUMNS - SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - int32_t offset = 0; - for (int32_t iCol = 0; iCol < taosArrayGetSize(pBlockData->aIdx); iCol++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); - SBlockCol blockCol = {0}; - - ASSERT(pColData->flag); - - if (pColData->flag == HAS_NONE) continue; - - blockCol.cid = pColData->cid; - blockCol.type = pColData->type; - blockCol.smaOn = pColData->smaOn; - blockCol.flag = pColData->flag; - blockCol.szOrigin = pColData->nData; - - if (pColData->flag != HAS_NULL) { - code = tsdbWriteColData(pColData, &blockCol, cmprAlg, ppBuf1, nBuf1, ppBuf2); - if (code) goto _err; - - blockCol.offset = offset; - offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); - nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); - } - - if (taosArrayPush(aBlockCol, &blockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - - // write - SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockData->suid, .uid = pBlockData->uid}; - code = tsdbWriteBlockDataImpl(pWriter->pLastFD, hdr, aBlockCol, *ppBuf1, nBuf1, ppBuf2, &pBlockL->szBlockCol); - if (code) goto _err; - - pBlockL->szBlock = pBlockL->szBlockCol + sizeof(TSCKSUM) + nBuf1; - pWriter->fLast.size += pBlockL->szBlock; - - tFree(pBuf1); - tFree(pBuf2); - return code; - -_err: - tFree(pBuf1); - tFree(pBuf2); + tFree(pBuf); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 13d293f86b..72d9175078 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -217,31 +217,30 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) { // SBlock ====================================================== void tBlockReset(SBlock *pBlock) { - *pBlock = - (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVersion = VERSION_MAX, .maxVersion = VERSION_MIN}; + *pBlock = (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; } int32_t tPutBlock(uint8_t *p, void *ph) { int32_t n = 0; SBlock *pBlock = (SBlock *)ph; - n += tPutTSDBKEY(p ? p + n : p, &pBlock->minKey); - n += tPutTSDBKEY(p ? p + n : p, &pBlock->maxKey); + n += tPutI64v(p ? p + n : p, pBlock->minKey.version); + n += tPutI64(p ? p + n : p, pBlock->minKey.ts); + n += tPutI64v(p ? p + n : p, pBlock->maxKey.version); + n += tPutI64(p ? p + n : p, pBlock->maxKey.ts); n += tPutI64v(p ? p + n : p, pBlock->minVer); n += tPutI64v(p ? p + n : p, pBlock->maxVer); n += tPutI32v(p ? p + n : p, pBlock->nRow); n += tPutI8(p ? p + n : p, pBlock->hasDup); n += tPutI8(p ? p + n : p, pBlock->nSubBlock); for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].nRow); - n += tPutI8(p ? p + n : p, pBlock->aSubBlock[iSubBlock].cmprAlg); n += tPutI64v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].offset); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szBlockCol); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szVersion); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szTSKEY); n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szBlock); - n += tPutI64v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].sOffset); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].nSma); + n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szKey); + } + if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { + n += tPutI64v(p ? p + n : p, pBlock->smaInfo.offset); + n += tPutI32v(p ? p + n : p, pBlock->smaInfo.size); } return n; @@ -251,23 +250,26 @@ int32_t tGetBlock(uint8_t *p, void *ph) { int32_t n = 0; SBlock *pBlock = (SBlock *)ph; - n += tGetTSDBKEY(p + n, &pBlock->minKey); - n += tGetTSDBKEY(p + n, &pBlock->maxKey); + n += tGetI64v(p + n, &pBlock->minKey.version); + n += tGetI64(p + n, &pBlock->minKey.ts); + n += tGetI64v(p + n, &pBlock->maxKey.version); + n += tGetI64(p + n, &pBlock->maxKey.ts); n += tGetI64v(p + n, &pBlock->minVer); n += tGetI64v(p + n, &pBlock->maxVer); n += tGetI32v(p + n, &pBlock->nRow); n += tGetI8(p + n, &pBlock->hasDup); n += tGetI8(p + n, &pBlock->nSubBlock); for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].nRow); - n += tGetI8(p + n, &pBlock->aSubBlock[iSubBlock].cmprAlg); n += tGetI64v(p + n, &pBlock->aSubBlock[iSubBlock].offset); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szBlockCol); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szVersion); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szTSKEY); n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szBlock); - n += tGetI64v(p + n, &pBlock->aSubBlock[iSubBlock].sOffset); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].nSma); + n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szKey); + } + if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { + n += tGetI64v(p + n, &pBlock->smaInfo.offset); + n += tGetI32v(p + n, &pBlock->smaInfo.size); + } else { + pBlock->smaInfo.offset = 0; + pBlock->smaInfo.size = 0; } return n; @@ -290,8 +292,9 @@ bool tBlockHasSma(SBlock *pBlock) { if (pBlock->nSubBlock > 1) return false; if (pBlock->hasDup) return false; - return pBlock->aSubBlock[0].nSma > 0; + return pBlock->smaInfo.size > 0; } + // SBlockL ====================================================== int32_t tPutBlockL(uint8_t *p, void *ph) { int32_t n = 0; @@ -303,13 +306,9 @@ int32_t tPutBlockL(uint8_t *p, void *ph) { n += tPutI64v(p ? p + n : p, pBlockL->minVer); n += tPutI64v(p ? p + n : p, pBlockL->maxVer); n += tPutI32v(p ? p + n : p, pBlockL->nRow); - n += tPutI64v(p ? p + n : p, pBlockL->offset); - n += tPutI8(p ? p + n : p, pBlockL->cmprAlg); - n += tPutI32v(p ? p + n : p, pBlockL->szBlockCol); - n += tPutI32v(p ? p + n : p, pBlockL->szUid); - n += tPutI32v(p ? p + n : p, pBlockL->szVer); - n += tPutI32v(p ? p + n : p, pBlockL->szTSKEY); - n += tPutI32v(p ? p + n : p, pBlockL->szBlock); + n += tPutI64v(p ? p + n : p, pBlockL->bInfo.offset); + n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szBlock); + n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szKey); return n; } @@ -324,13 +323,9 @@ int32_t tGetBlockL(uint8_t *p, void *ph) { n += tGetI64v(p + n, &pBlockL->minVer); n += tGetI64v(p + n, &pBlockL->maxVer); n += tGetI32v(p + n, &pBlockL->nRow); - n += tGetI64v(p + n, &pBlockL->offset); - n += tGetI8(p + n, &pBlockL->cmprAlg); - n += tGetI32v(p + n, &pBlockL->szBlockCol); - n += tGetI32v(p + n, &pBlockL->szUid); - n += tGetI32v(p + n, &pBlockL->szVer); - n += tGetI32v(p + n, &pBlockL->szTSKEY); - n += tGetI32v(p + n, &pBlockL->szBlock); + n += tGetI64v(p + n, &pBlockL->bInfo.offset); + n += tGetI32v(p + n, &pBlockL->bInfo.szBlock); + n += tGetI32v(p + n, &pBlockL->bInfo.szKey); return n; } @@ -346,15 +341,25 @@ int32_t tPutBlockCol(uint8_t *p, void *ph) { n += tPutI8(p ? p + n : p, pBlockCol->type); n += tPutI8(p ? p + n : p, pBlockCol->smaOn); n += tPutI8(p ? p + n : p, pBlockCol->flag); + n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin); if (pBlockCol->flag != HAS_NULL) { - n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin); - n += tPutI32v(p ? p + n : p, pBlockCol->szBitmap); - n += tPutI32v(p ? p + n : p, pBlockCol->szOffset); - n += tPutI32v(p ? p + n : p, pBlockCol->szValue); + if (pBlockCol->flag != HAS_VALUE) { + n += tPutI32v(p ? p + n : p, pBlockCol->szBitmap); + } + + if (IS_VAR_DATA_TYPE(pBlockCol->type)) { + n += tPutI32v(p ? p + n : p, pBlockCol->szOffset); + } + + if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) { + n += tPutI32v(p ? p + n : p, pBlockCol->szValue); + } + n += tPutI32v(p ? p + n : p, pBlockCol->offset); } +_exit: return n; } @@ -366,14 +371,28 @@ int32_t tGetBlockCol(uint8_t *p, void *ph) { n += tGetI8(p + n, &pBlockCol->type); n += tGetI8(p + n, &pBlockCol->smaOn); n += tGetI8(p + n, &pBlockCol->flag); + n += tGetI32v(p + n, &pBlockCol->szOrigin); ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE)); + pBlockCol->szBitmap = 0; + pBlockCol->szOffset = 0; + pBlockCol->szValue = 0; + pBlockCol->offset = 0; + if (pBlockCol->flag != HAS_NULL) { - n += tGetI32v(p + n, &pBlockCol->szOrigin); - n += tGetI32v(p + n, &pBlockCol->szBitmap); - n += tGetI32v(p + n, &pBlockCol->szOffset); - n += tGetI32v(p + n, &pBlockCol->szValue); + if (pBlockCol->flag != HAS_VALUE) { + n += tGetI32v(p + n, &pBlockCol->szBitmap); + } + + if (IS_VAR_DATA_TYPE(pBlockCol->type)) { + n += tGetI32v(p + n, &pBlockCol->szOffset); + } + + if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) { + n += tGetI32v(p + n, &pBlockCol->szValue); + } + n += tGetI32v(p + n, &pBlockCol->offset); } @@ -1650,23 +1669,36 @@ int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cm taosArrayClear(pDiskData->aBlockCol); pDiskData->nBuf = 0; - // uid - if (pDiskData->uid == 0) { - code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pDiskData->pUid, &pDiskData->szUid, &pDiskData->pBuf); + { + pDiskData->ppKey = tDiskDataAllocBuf(pDiskData); + if (pDiskData->ppKey == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + int32_t n = 0; + // uid + if (pDiskData->uid == 0) { + code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, pDiskData->ppKey, n, &pDiskData->szUid, NULL); + if (code) goto _exit; + } else { + pDiskData->szUid = 0; + } + n += pDiskData->szUid; + + // version + code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, pDiskData->ppKey, n, &pDiskData->szVer, NULL); + if (code) goto _exit; + n += pDiskData->szVer; + + // tskey + code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, + cmprAlg, pDiskData->ppKey, &pDiskData->szKey, NULL); if (code) goto _exit; } - // version - code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &pDiskData->pVer, &pDiskData->szVer, &pDiskData->pBuf); - if (code) goto _exit; - - // tskey - code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &pDiskData->pKey, &pDiskData->szKey, &pDiskData->pBuf); - if (code) goto _exit; - // columns int32_t offset = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { @@ -1689,7 +1721,7 @@ int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cm } // compress - code = tsdbCmprColData(pColData, cmprAlg, &blockCol); + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, NULL); if (code) goto _exit; // update offset @@ -1709,6 +1741,41 @@ _exit: int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); +// SDiskDataHdr ============================== +int32_t tPutDiskDataHdr(uint8_t *p, void *ph) { + int32_t n = 0; + SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; + + n += tPutU32(p ? p + n : p, pHdr->delimiter); + n += tPutI64(p ? p + n : p, pHdr->suid); + n += tPutI64(p ? p + n : p, pHdr->uid); + n += tPutI32v(p ? p + n : p, pHdr->szUid); + n += tPutI32v(p ? p + n : p, pHdr->szVer); + n += tPutI32v(p ? p + n : p, pHdr->szKey); + n += tPutI32v(p ? p + n : p, pHdr->szBlkCol); + n += tPutI32v(p ? p + n : p, pHdr->nRow); + n += tPutI8(p ? p + n : p, pHdr->cmprAlg); + + return n; +} + +int32_t tGetDiskDataHdr(uint8_t *p, void *ph) { + int32_t n = 0; + SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; + + n += tGetU32(p + n, &pHdr->delimiter); + n += tGetI64(p + n, &pHdr->suid); + n += tGetI64(p + n, &pHdr->uid); + n += tGetI32v(p + n, &pHdr->szUid); + n += tGetI32v(p + n, &pHdr->szVer); + n += tGetI32v(p + n, &pHdr->szKey); + n += tGetI32v(p + n, &pHdr->szBlkCol); + n += tGetI32v(p + n, &pHdr->nRow); + n += tGetI8(p + n, &pHdr->cmprAlg); + + return n; +} + // ALGORITHM ============================== void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { SColVal colVal; From 85907bc2696c173345ee76e5332613c0963f8944 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 02:03:20 +0000 Subject: [PATCH 014/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 08331b1327..bb08a345c2 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -62,6 +62,8 @@ typedef struct SRowIter SRowIter; typedef struct STsdbFS STsdbFS; typedef struct SRowMerger SRowMerger; typedef struct STsdbReadSnap STsdbReadSnap; +typedef struct SBlockInfo SBlockInfo; +typedef struct SSmaInfo SSmaInfo; #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 @@ -411,16 +413,16 @@ typedef struct { uint8_t **ppData; } SBlockCol; -typedef struct { +struct SBlockInfo { int64_t offset; // block data offset int32_t szBlock; int32_t szKey; -} SBlockInfo; +}; -typedef struct { +struct SSmaInfo { int64_t offset; int32_t size; -} SSmaInfo; +}; struct SBlock { TSDBKEY minKey; From 04386a3003fee0c41c95313a62730dea22f3e456 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 07:45:24 +0000 Subject: [PATCH 015/194] more work --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index cd94762c7b..aa76ab1e27 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -530,8 +530,10 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { } // write - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock++], - &block.smaInfo, pCommitter->cmprAlg, 0, NULL); + block.nSubBlock++; + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock - 1], + ((block.nSubBlock == 1) && block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0, + NULL); if (code) goto _err; // put SBlock @@ -562,7 +564,7 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { blockL.maxVer = VERSION_MIN; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { blockL.minVer = TMIN(blockL.minVer, pBlockData->aVersion[iRow]); - blockL.maxVer = TMIN(blockL.maxVer, pBlockData->aVersion[iRow]); + blockL.maxVer = TMAX(blockL.maxVer, pBlockData->aVersion[iRow]); } blockL.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0]; blockL.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; From 3324f4e5b7a979323984c348a512263ef8537e0a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 12:04:42 +0000 Subject: [PATCH 016/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 17 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 6 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 23 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 226 ++++++++---------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 12 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 51 ++-- 7 files changed, 162 insertions(+), 177 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index bb08a345c2..bd24b45ccd 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -183,6 +183,8 @@ void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *m int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now); int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline); void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg); +int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); +int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); @@ -238,9 +240,9 @@ int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile); int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf); -int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBuf, SBlockIdx *pBlockIdx); -int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf); +int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); +int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx); +int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast, uint8_t **ppBuf); @@ -248,9 +250,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); -int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf); -int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf); -int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData, uint8_t **ppBuf); +int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); +int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); +int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, @@ -605,6 +607,9 @@ struct SDataFWriter { SLastFile fLast; SSmaFile fSma; + uint8_t *pBuf1; + uint8_t *pBuf2; + SDiskData dData; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index f03b02af27..1fc5eae13a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -465,13 +465,12 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (code) goto _err; // tMapDataReset(&state->blockIdxMap); - // code = tsdbReadBlockIdx(state->pDataFReader, &state->blockIdxMap, NULL); if (!state->aBlockIdx) { state->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); } else { taosArrayClear(state->aBlockIdx); } - code = tsdbReadBlockIdx(state->pDataFReader, state->aBlockIdx, NULL); + code = tsdbReadBlockIdx(state->pDataFReader, state->aBlockIdx); if (code) goto _err; /* if (state->pBlockIdx) { */ @@ -487,8 +486,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } tMapDataReset(&state->blockMap); - code = tsdbReadBlock(state->pDataFReader, state->pBlockIdx, &state->blockMap, NULL); - /* code = tsdbReadBlock(state->pDataFReader, &state->blockIdx, &state->blockMap, NULL); */ + code = tsdbReadBlock(state->pDataFReader, state->pBlockIdx, &state->blockMap); if (code) goto _err; state->nBlock = state->blockMap.nItem; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index aa76ab1e27..5470e43400 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -339,7 +339,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); + code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); if (code) goto _exit; ASSERT(pCommitter->dReader.mBlock.nItem > 0); @@ -370,7 +370,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { if (code) goto _err; // data - code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx, NULL); + code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx); if (code) goto _err; pCommitter->dReader.iBlockIdx = 0; @@ -378,8 +378,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - code = - tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock, NULL); + code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); if (code) goto _err; } else { pCommitter->dReader.pBlockIdx = NULL; @@ -387,7 +386,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { tBlockDataReset(&pCommitter->dReader.bData); // last - code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL, NULL); + code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL); if (code) goto _err; pCommitter->dReader.iBlockL = -1; @@ -506,7 +505,7 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { tBlockReset(&block); // as a new block } - // statistic + // info block.nRow += pBlockData->nRow; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; @@ -532,7 +531,7 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { // write block.nSubBlock++; code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock - 1], - ((block.nSubBlock == 1) && block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0, + ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0, NULL); if (code) goto _err; @@ -557,7 +556,7 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { ASSERT(pBlockData->nRow > 0); - // statistic + // info blockL.suid = pBlockData->suid; blockL.nRow = pBlockData->nRow; blockL.minVer = VERSION_MAX; @@ -1041,7 +1040,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { // end if (pCommitter->dWriter.mBlock.nItem > 0) { SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid}; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, NULL, &blockIdx); + code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); if (code) goto _err; if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { @@ -1067,11 +1066,11 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; // write aBlockIdx - code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx, NULL); + code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); if (code) goto _err; // write aBlockL - code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL, NULL); + code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL); if (code) goto _err; // update file header @@ -1107,7 +1106,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { if (pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) >= 0) break; SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, NULL, &blockIdx); + code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); if (code) goto _err; if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 84ecaa605d..df61d3d4c9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -545,7 +545,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, SArray* aBlockIdx = taosArrayInit(8, sizeof(SBlockIdx)); int64_t st = taosGetTimestampUs(); - int32_t code = tsdbReadBlockIdx(pFileReader, aBlockIdx, NULL); + int32_t code = tsdbReadBlockIdx(pFileReader, aBlockIdx); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -617,7 +617,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_ STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(int64_t)); tMapDataReset(&pScanInfo->mapData); - tsdbReadBlock(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData, NULL); + tsdbReadBlock(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); size += pScanInfo->mapData.nData; for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index c542ab5583..4db0333890 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -441,6 +441,9 @@ struct SDataFReader { TdFilePtr pDataFD; TdFilePtr pLastFD; TdFilePtr pSmaFD; + + uint8_t *pBuf1; + uint8_t *pBuf2; }; int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { @@ -523,6 +526,8 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { goto _err; } + tFree((*ppReader)->pBuf1); + tFree((*ppReader)->pBuf2); taosMemoryFree(*ppReader); _exit: @@ -534,14 +539,12 @@ _err: return code; } -int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf) { - int32_t code = 0; - int64_t offset = pReader->pSet->pHeadF->offset; - int64_t size = pReader->pSet->pHeadF->loffset - offset; - uint8_t *pBuf = NULL; - int64_t n; - uint32_t delimiter; - SBlockIdx blockIdx; +int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { + int32_t code = 0; + int64_t offset = pReader->pSet->pHeadF->offset; + int64_t size = pReader->pSet->pHeadF->loffset - offset; + int64_t n; + uint32_t delimiter; taosArrayClear(aBlockIdx); if (size == 0) { @@ -549,8 +552,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB } // alloc - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; // seek @@ -560,7 +562,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB } // read - n = taosReadFile(pReader->pHeadFD, *ppBuf, size); + n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -570,18 +572,19 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode n = 0; - n = tGetU32(*ppBuf + n, &delimiter); + n = tGetU32(pReader->pBuf1 + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); while (n < size - sizeof(TSCKSUM)) { - n += tGetBlockIdx(*ppBuf + n, &blockIdx); + SBlockIdx blockIdx; + n += tGetBlockIdx(pReader->pBuf1 + n, &blockIdx); if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -591,24 +594,20 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppB ASSERT(n + sizeof(TSCKSUM) == size); - tFree(pBuf); _exit: return code; _err: tsdbError("vgId:%d, read block idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } -int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) { +int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { int32_t code = 0; int64_t offset = pReader->pSet->pHeadF->loffset; int64_t size = pReader->pSet->pHeadF->size - offset; int64_t n; uint32_t delimiter; - uint8_t *pBuf = NULL; - SBlockL blockl; taosArrayClear(aBlockL); if (size == 0) { @@ -616,8 +615,7 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) } // alloc - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; // seek @@ -627,7 +625,7 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) } // read - n = taosReadFile(pReader->pHeadFD, *ppBuf, size); + n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -637,18 +635,19 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode n = 0; - n = tGetU32(*ppBuf + n, &delimiter); + n = tGetU32(pReader->pBuf1 + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); while (n < size - sizeof(TSCKSUM)) { - n += tGetBlockL(*ppBuf + n, &blockl); + SBlockL blockl; + n += tGetBlockL(pReader->pBuf1 + n, &blockl); if (taosArrayPush(aBlockL, &blockl) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -658,29 +657,23 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL, uint8_t **ppBuf) ASSERT(n + sizeof(TSCKSUM) == size); - tFree(pBuf); _exit: return code; _err: tsdbError("vgId:%d read blockl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } -int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock, uint8_t **ppBuf) { - int32_t code = 0; - int64_t offset = pBlockIdx->offset; - int64_t size = pBlockIdx->size; - uint8_t *pBuf = NULL; - int64_t n; - int64_t tn; - SDiskDataHdr hdr; - - if (!ppBuf) ppBuf = &pBuf; +int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock) { + int32_t code = 0; + int64_t offset = pBlockIdx->offset; + int64_t size = pBlockIdx->size; + int64_t n; + int64_t tn; // alloc - code = tRealloc(ppBuf, size); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; // seek @@ -690,7 +683,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl } // read - n = taosReadFile(pReader->pHeadFD, *ppBuf, size); + n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -700,19 +693,19 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode - hdr = *(SDiskDataHdr *)(*ppBuf); - ASSERT(hdr.delimiter == TSDB_FILE_DLMT); - ASSERT(hdr.suid == pBlockIdx->suid); - ASSERT(hdr.uid == pBlockIdx->uid); + n = 0; - n = sizeof(hdr); - tn = tGetMapData(*ppBuf + n, mBlock); + uint32_t delimiter; + n += tGetU32(pReader->pBuf1 + n, &delimiter); + ASSERT(delimiter == TSDB_FILE_DLMT); + + tn = tGetMapData(pReader->pBuf1 + n, mBlock); if (tn < 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -720,12 +713,10 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl n += tn; ASSERT(n + sizeof(TSCKSUM) == size); - tFree(pBuf); return code; _err: tsdbError("vgId:%d, read block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } @@ -1642,6 +1633,8 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { } tDiskDataClear(&(*ppWriter)->dData); + tFree((*ppWriter)->pBuf1); + tFree((*ppWriter)->pBuf2); taosMemoryFree(*ppWriter); _exit: *ppWriter = NULL; @@ -1732,10 +1725,9 @@ _err: return code; } -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf) { +int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { int32_t code = 0; SHeadFile *pHeadFile = &pWriter->fHead; - uint8_t *pBuf = NULL; int64_t size; int64_t n; @@ -1746,29 +1738,28 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **pp } // prepare - size = tPutU32(NULL, TSDB_FILE_DLMT); + size = sizeof(uint32_t); for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx)); } size += sizeof(TSCKSUM); // alloc - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + code = tRealloc(&pWriter->pBuf1, size); if (code) goto _err; // build n = 0; - n = tPutU32(*ppBuf + n, TSDB_FILE_DLMT); + n = tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - n += tPutBlockIdx(*ppBuf + n, taosArrayGet(aBlockIdx, iBlockIdx)); + n += tPutBlockIdx(pWriter->pBuf1 + n, taosArrayGet(aBlockIdx, iBlockIdx)); } - taosCalcChecksumAppend(0, *ppBuf, size); + taosCalcChecksumAppend(0, pWriter->pBuf1, size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, *ppBuf, size); + n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1779,44 +1770,36 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **pp pHeadFile->size += size; _exit: - tFree(pBuf); return code; _err: tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } -int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, uint8_t **ppBuf, SBlockIdx *pBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; - uint8_t *pBuf = NULL; - int64_t size; - int64_t n; +int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBlockIdx) { + int32_t code = 0; + SHeadFile *pHeadFile = &pWriter->fHead; + int64_t size; + int64_t n; ASSERT(mBlock->nItem > 0); - // prepare - size = sizeof(SDiskDataHdr) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); - // alloc - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + size = sizeof(uint32_t) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); + code = tRealloc(&pWriter->pBuf1, size); if (code) goto _err; // build n = 0; - *(SDiskDataHdr *)(*ppBuf) = hdr; - n += sizeof(hdr); - n += tPutMapData(*ppBuf + n, mBlock); - taosCalcChecksumAppend(0, *ppBuf, size); + n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); + n += tPutMapData(pWriter->pBuf1 + n, mBlock); + taosCalcChecksumAppend(0, pWriter->pBuf1, size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, *ppBuf, size); + n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1827,21 +1810,18 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, uint8_t **ppBuf, pBlockIdx->size = size; pHeadFile->size += size; - tFree(pBuf); tsdbTrace("vgId:%d, write block, offset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), pBlockIdx->offset, pBlockIdx->size); return code; _err: - tFree(pBuf); tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) { +int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { int32_t code = 0; SHeadFile *pHeadFile = &pWriter->fHead; - uint8_t *pBuf = NULL; int64_t size; int64_t n; @@ -1852,29 +1832,28 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) } // size - size = sizeof(uint32_t); + size = sizeof(uint32_t); // TSDB_FILE_DLMT for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { size += tPutBlockL(NULL, taosArrayGet(aBlockL, iBlockL)); } size += sizeof(TSCKSUM); // alloc - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + code = tRealloc(&pWriter->pBuf1, size); if (code) goto _err; // encode n = 0; - n += tPutU32(*ppBuf + n, TSDB_FILE_DLMT); + n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { - n += tPutBlockL(*ppBuf + n, taosArrayGet(aBlockL, iBlockL)); + n += tPutBlockL(pWriter->pBuf1 + n, taosArrayGet(aBlockL, iBlockL)); } - taosCalcChecksumAppend(0, *ppBuf, size); + taosCalcChecksumAppend(0, pWriter->pBuf1, size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, *ppBuf, size); + n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1885,12 +1864,10 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL, uint8_t **ppBuf) pHeadFile->size += size; _exit: - tFree(pBuf); return code; _err: tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } @@ -1918,47 +1895,49 @@ static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { pBlock->nRow += pBlockData->nRow; } -static int32_t tsdbWriteBlockSma(TdFilePtr pFD, SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t **ppBuf) { - int32_t code = 0; - int64_t n; - SColData *pColData; +static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { + int32_t code = 0; - // prepare - pSubBlock->nSma = 0; + pSmaInfo->offset = 0; + pSmaInfo->size = 0; + + // encode for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - if (IS_VAR_DATA_TYPE(pColData->type) || (!pColData->smaOn)) continue; + if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue; - pSubBlock->nSma++; + SColumnDataAgg sma; + tsdbCalcColDataSMA(pColData, &sma); + + code = tRealloc(&pWriter->pBuf1, pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); + if (code) goto _err; + pSmaInfo->size += tPutColumnDataAgg(pWriter->pBuf1 + pSmaInfo->size, &sma); } - if (pSubBlock->nSma == 0) goto _exit; - - // calc - code = tRealloc(ppBuf, sizeof(SColumnDataAgg) * pSubBlock->nSma + sizeof(TSCKSUM)); - if (code) goto _err; - n = 0; - for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - if (IS_VAR_DATA_TYPE(pColData->type) || (!pColData->smaOn)) continue; - - tsdbCalcColDataSMA(pColData, &((SColumnDataAgg *)(*ppBuf))[n]); - n++; - } - taosCalcChecksumAppend(0, *ppBuf, sizeof(SColumnDataAgg) * pSubBlock->nSma + sizeof(TSCKSUM)); // write - n = taosWriteFile(pFD, *ppBuf, sizeof(SColumnDataAgg) * pSubBlock->nSma + sizeof(TSCKSUM)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (pSmaInfo->size) { + int32_t size = pSmaInfo->size + sizeof(TSCKSUM); + + code = tRealloc(&pWriter->pBuf1, size); + if (code) goto _err; + + taosCalcChecksumAppend(0, pWriter->pBuf1, size); + + int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->pBuf1, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pSmaInfo->offset = pWriter->fSma.size; + pWriter->fSma.size += size; } -_exit: return code; _err: + tsdbError("vgId:%d tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } @@ -2036,12 +2015,11 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { } -// ================= SMA ==================== -_write_sma: - if (toLast) goto _exit; - if (pSmaInfo == NULL) goto _exit; - - // TODO + // ================= SMA ==================== + if (pSmaInfo) { + code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo); + if (code) goto _err; + } _exit: tFree(pBuf); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index eec7a3e4a6..2b5ea030b2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -58,7 +58,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { if (code) goto _err; // SBlockIdx - code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx, NULL); + code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); if (code) goto _err; pReader->iBlockIdx = 0; @@ -78,7 +78,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); pReader->iBlockIdx++; - code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock, NULL); + code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); if (code) goto _err; pReader->iBlock = 0; @@ -552,7 +552,7 @@ _err: static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* pBlockIdx) { int32_t code = 0; - code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL); + code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock); if (code) goto _err; // SBlockData @@ -791,7 +791,7 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { } if (pWriter->pBlockIdx) { - code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock, NULL); + code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock); if (code) goto _err; } else { tMapDataReset(&pWriter->mBlock); @@ -843,7 +843,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { pWriter->iBlockIdx++; } - code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW, NULL); + code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW); if (code) goto _err; code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet); @@ -897,7 +897,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet); if (code) goto _err; - code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx, NULL); + code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx); if (code) goto _err; } else { ASSERT(pWriter->pDataFReader == NULL); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 72d9175078..0ec922f20e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -152,25 +152,6 @@ int32_t tTABLEIDCmprFn(const void *p1, const void *p2) { return 0; } -// TSDBKEY ====================================================== -static FORCE_INLINE int32_t tPutTSDBKEY(uint8_t *p, TSDBKEY *pKey) { - int32_t n = 0; - - n += tPutI64v(p ? p + n : p, pKey->version); - n += tPutI64(p ? p + n : p, pKey->ts); - - return n; -} - -static FORCE_INLINE int32_t tGetTSDBKEY(uint8_t *p, TSDBKEY *pKey) { - int32_t n = 0; - - n += tGetI64v(p + n, &pKey->version); - n += tGetI64(p + n, &pKey->ts); - - return n; -} - // SBlockIdx ====================================================== int32_t tPutBlockIdx(uint8_t *p, void *ph) { int32_t n = 0; @@ -225,9 +206,9 @@ int32_t tPutBlock(uint8_t *p, void *ph) { SBlock *pBlock = (SBlock *)ph; n += tPutI64v(p ? p + n : p, pBlock->minKey.version); - n += tPutI64(p ? p + n : p, pBlock->minKey.ts); + n += tPutI64v(p ? p + n : p, pBlock->minKey.ts); n += tPutI64v(p ? p + n : p, pBlock->maxKey.version); - n += tPutI64(p ? p + n : p, pBlock->maxKey.ts); + n += tPutI64v(p ? p + n : p, pBlock->maxKey.ts); n += tPutI64v(p ? p + n : p, pBlock->minVer); n += tPutI64v(p ? p + n : p, pBlock->maxVer); n += tPutI32v(p ? p + n : p, pBlock->nRow); @@ -251,9 +232,9 @@ int32_t tGetBlock(uint8_t *p, void *ph) { SBlock *pBlock = (SBlock *)ph; n += tGetI64v(p + n, &pBlock->minKey.version); - n += tGetI64(p + n, &pBlock->minKey.ts); + n += tGetI64v(p + n, &pBlock->minKey.ts); n += tGetI64v(p + n, &pBlock->maxKey.version); - n += tGetI64(p + n, &pBlock->maxKey.ts); + n += tGetI64v(p + n, &pBlock->maxKey.ts); n += tGetI64v(p + n, &pBlock->minVer); n += tGetI64v(p + n, &pBlock->maxVer); n += tGetI32v(p + n, &pBlock->nRow); @@ -1777,6 +1758,30 @@ int32_t tGetDiskDataHdr(uint8_t *p, void *ph) { } // ALGORITHM ============================== +int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { + int32_t n = 0; + + n += tPutI16v(p ? p + n : p, pColAgg->colId); + n += tPutI16v(p ? p + n : p, pColAgg->numOfNull); + n += tPutI64(p ? p + n : p, pColAgg->sum); + n += tPutI64(p ? p + n : p, pColAgg->max); + n += tPutI64(p ? p + n : p, pColAgg->min); + + return n; +} + +int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { + int32_t n = 0; + + n += tGetI16v(p + n, &pColAgg->colId); + n += tGetI16v(p + n, &pColAgg->numOfNull); + n += tGetI64(p + n, &pColAgg->sum); + n += tGetI64(p + n, &pColAgg->max); + n += tGetI64(p + n, &pColAgg->min); + + return n; +} + void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { SColVal colVal; SColVal *pColVal = &colVal; From 3436d52316d3882db8b0dfd3b96ca41ae5f7def7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 12:23:29 +0000 Subject: [PATCH 017/194] more refact --- source/dnode/vnode/src/inc/tsdb.h | 12 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 5 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 10 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 153 +++++++----------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 14 +- 6 files changed, 78 insertions(+), 120 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index bd24b45ccd..60727b5e8e 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -265,14 +265,14 @@ int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *p // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf, SDelIdx *pDelIdx); -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx, uint8_t **ppBuf); +int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx); +int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx); int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter); // SDelFReader -int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb, uint8_t **ppBuf); +int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFReaderClose(SDelFReader **ppReader); -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf); -int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf); +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); +int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap); void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap); @@ -583,6 +583,8 @@ struct SDelFWriter { STsdb *pTsdb; SDelFile fDel; TdFilePtr pWriteH; + + uint8_t *pBuf1; }; struct SDiskData { diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 1fc5eae13a..5ad1f3bcb9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -317,7 +317,7 @@ static int32_t getTableDelDataFromDelIdx(SDelFReader *pDelReader, SDelIdx *pDelI int32_t code = 0; if (pDelIdx) { - code = tsdbReadDelData(pDelReader, pDelIdx, aDelData, NULL); + code = tsdbReadDelData(pDelReader, pDelIdx, aDelData); } return code; @@ -388,8 +388,7 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t SDelIdx idx = {.suid = suid, .uid = uid}; // tMapDataReset(&delIdxMap); - // code = tsdbReadDelIdx(pDelFReader, &delIdxMap, NULL); - code = tsdbReadDelIdx(pDelFReader, pDelIdxArray, NULL); + code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); if (code) goto _err; // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 5470e43400..3bb78af44a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -181,10 +181,10 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { SDelFile *pDelFileR = pCommitter->fs.pDelFile; if (pDelFileR) { - code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb, NULL); + code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb); if (code) goto _err; - code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx, NULL); + code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx); if (code) goto _err; } @@ -221,7 +221,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel suid = pDelIdx->suid; uid = pDelIdx->uid; - code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, NULL); + code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData); if (code) goto _err; } else { taosArrayClear(pCommitter->aDelData); @@ -241,7 +241,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel } // write - code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, NULL, &delIdx); + code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, &delIdx); if (code) goto _err; // put delIdx @@ -262,7 +262,7 @@ static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; - code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN, NULL); + code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN); if (code) goto _err; code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index df61d3d4c9..4523dda2b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1618,7 +1618,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* goto _err; } - code = tsdbReadDelIdx(pDelFReader, aDelIdx, NULL); + code = tsdbReadDelIdx(pDelFReader, aDelIdx); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(aDelIdx); tsdbDelFReaderClose(&pDelFReader); @@ -1629,7 +1629,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* SDelIdx* pIdx = taosArraySearch(aDelIdx, &idx, tCmprDelIdx, TD_EQ); if (pIdx != NULL) { - code = tsdbReadDelData(pDelFReader, pIdx, pDelData, NULL); + code = tsdbReadDelData(pDelFReader, pIdx, pDelData); } taosArrayDestroy(aDelIdx); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 4db0333890..8ee8b085c0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -76,6 +76,9 @@ int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { goto _err; } + tFree(pWriter->pBuf1); + taosMemoryFree(pWriter); + *ppWriter = NULL; return code; @@ -84,39 +87,34 @@ _err: return code; } -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf, SDelIdx *pDelIdx) { - int32_t code = 0; - uint8_t *pBuf = NULL; - int64_t size; - int64_t n; - SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pDelIdx->suid, .uid = pDelIdx->uid}; - - if (!ppBuf) ppBuf = &pBuf; +int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) { + int32_t code = 0; + int64_t size; + int64_t n; // prepare - size = sizeof(hdr); + size = sizeof(uint32_t); for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData)); } size += sizeof(TSCKSUM); // alloc - code = tRealloc(ppBuf, size); + code = tRealloc(&pWriter->pBuf1, size); if (code) goto _err; // build n = 0; - *(SDiskDataHdr *)(*ppBuf) = hdr; - n += sizeof(hdr); + n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - n += tPutDelData(*ppBuf + n, taosArrayGet(aDelData, iDelData)); + n += tPutDelData(pWriter->pBuf1 + n, taosArrayGet(aDelData, iDelData)); } - taosCalcChecksumAppend(0, *ppBuf, size); + taosCalcChecksumAppend(0, pWriter->pBuf1, size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pWriteH, *ppBuf, size); + n = taosWriteFile(pWriter->pWriteH, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -129,48 +127,42 @@ int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf pDelIdx->size = size; pWriter->fDel.size += size; - tFree(pBuf); return code; _err: tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx, uint8_t **ppBuf) { +int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { int32_t code = 0; int64_t size; int64_t n; - uint8_t *pBuf = NULL; SDelIdx *pDelIdx; - if (!ppBuf) ppBuf = &pBuf; - // prepare - size = 0; - size += tPutU32(NULL, TSDB_FILE_DLMT); + size = sizeof(uint32_t); for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx)); } size += sizeof(TSCKSUM); // alloc - code = tRealloc(ppBuf, size); + code = tRealloc(&pWriter->pBuf1, size); if (code) goto _err; // build n = 0; - n += tPutU32(*ppBuf + n, TSDB_FILE_DLMT); + n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - n += tPutDelIdx(*ppBuf + n, taosArrayGet(aDelIdx, iDelIdx)); + n += tPutDelIdx(pWriter->pBuf1 + n, taosArrayGet(aDelIdx, iDelIdx)); } - taosCalcChecksumAppend(0, *ppBuf, size); + taosCalcChecksumAppend(0, pWriter->pBuf1, size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pWriteH, *ppBuf, size); + n = taosWriteFile(pWriter->pWriteH, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -180,12 +172,10 @@ int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx, uint8_t **ppBuf) pWriter->fDel.offset = pWriter->fDel.size; pWriter->fDel.size += size; - tFree(pBuf); return code; _err: tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } @@ -225,9 +215,11 @@ struct SDelFReader { STsdb *pTsdb; SDelFile fDel; TdFilePtr pReadH; + + uint8_t *pBuf1; }; -int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb, uint8_t **ppBuf) { +int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; char fname[TSDB_FILENAME_LEN]; SDelFReader *pDelFReader; @@ -252,32 +244,6 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb goto _err; } -#if 0 - // load and check hdr if buffer is given - if (ppBuf) { - code = tRealloc(ppBuf, TSDB_FHDR_SIZE); - if (code) { - goto _err; - } - - n = taosReadFile(pDelFReader->pReadH, *ppBuf, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < TSDB_FHDR_SIZE) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - if (!taosCheckChecksumWhole(*ppBuf, TSDB_FHDR_SIZE)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // TODO: check the content - } -#endif - _exit: *ppReader = pDelFReader; return code; @@ -297,6 +263,7 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { code = TAOS_SYSTEM_ERROR(errno); goto _exit; } + tFree(pReader->pBuf1); taosMemoryFree(pReader); } *ppReader = NULL; @@ -305,16 +272,13 @@ _exit: return code; } -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf) { - int32_t code = 0; - int64_t offset = pDelIdx->offset; - int64_t size = pDelIdx->size; - int64_t n; - uint8_t *pBuf = NULL; - SDiskDataHdr *pHdr; - SDelData *pDelData = &(SDelData){0}; +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { + int32_t code = 0; + int64_t offset = pDelIdx->offset; + int64_t size = pDelIdx->size; + int64_t n; - if (!ppBuf) ppBuf = &pBuf; + taosArrayClear(aDelData); // seek if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { @@ -323,11 +287,11 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData } // alloc - code = tRealloc(ppBuf, size); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; // read - n = taosReadFile(pReader->pReadH, *ppBuf, size); + n = taosReadFile(pReader->pReadH, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -337,23 +301,21 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // // decode n = 0; - pHdr = (SDiskDataHdr *)(*ppBuf + n); - ASSERT(pHdr->delimiter == TSDB_FILE_DLMT); - ASSERT(pHdr->suid == pDelIdx->suid); - ASSERT(pHdr->uid == pDelIdx->uid); - n += sizeof(*pHdr); - taosArrayClear(aDelData); - while (n < size - sizeof(TSCKSUM)) { - n += tGetDelData(*ppBuf + n, pDelData); - if (taosArrayPush(aDelData, pDelData) == NULL) { + uint32_t delimiter; + n += tGetU32(pReader->pBuf1 + n, &delimiter); + while (n < size - sizeof(TSCKSUM)) { + SDelData delData; + n += tGetDelData(pReader->pBuf1 + n, &delData); + + if (taosArrayPush(aDelData, &delData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -361,25 +323,20 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData ASSERT(n == size - sizeof(TSCKSUM)); - tFree(pBuf); return code; _err: tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } -int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf) { - int32_t code = 0; - int32_t n; - int64_t offset = pReader->fDel.offset; - int64_t size = pReader->fDel.size - offset; - uint32_t delimiter; - uint8_t *pBuf = NULL; - SDelIdx *pDelIdx = &(SDelIdx){0}; +int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { + int32_t code = 0; + int32_t n; + int64_t offset = pReader->fDel.offset; + int64_t size = pReader->fDel.size - offset; - if (!ppBuf) ppBuf = &pBuf; + taosArrayClear(aDelIdx); // seek if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { @@ -388,11 +345,11 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf) { } // alloc - code = tRealloc(ppBuf, size); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; // read - n = taosReadFile(pReader->pReadH, *ppBuf, size); + n = taosReadFile(pReader->pReadH, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -402,21 +359,23 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf) { } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode n = 0; - n += tGetU32(*ppBuf + n, &delimiter); + uint32_t delimiter; + n += tGetU32(pReader->pBuf1 + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); - taosArrayClear(aDelIdx); while (n < size - sizeof(TSCKSUM)) { - n += tGetDelIdx(*ppBuf + n, pDelIdx); + SDelIdx delIdx; - if (taosArrayPush(aDelIdx, pDelIdx) == NULL) { + n += tGetDelIdx(pReader->pBuf1 + n, &delIdx); + + if (taosArrayPush(aDelIdx, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -424,12 +383,10 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf) { ASSERT(n == size - sizeof(TSCKSUM)); - tFree(pBuf); return code; _err: tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 2b5ea030b2..06a38303ec 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -177,7 +177,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { if (code) goto _err; // read index - code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx, NULL); + code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx); if (code) goto _err; pReader->iDelIdx = 0; @@ -193,7 +193,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->iDelIdx++; - code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData, NULL); + code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData); if (code) goto _err; int32_t size = 0; @@ -968,7 +968,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL); if (code) goto _err; - code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR, NULL); + code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR); if (code) goto _err; } @@ -997,7 +997,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 if (c < 0) { goto _new_del; } else { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL); + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); if (code) goto _err; pWriter->iDelIdx++; @@ -1027,7 +1027,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 } _write_del: - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx); + code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); if (code) goto _err; if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) { @@ -1058,11 +1058,11 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) { SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL); + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); if (code) goto _err; SDelIdx delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid}; - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx); + code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); if (code) goto _err; if (taosArrayPush(pWriter->aDelIdxR, &delIdx) == NULL) { From 04804e384aa0ea2162848db100c31cd20aef6b56 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 12:24:46 +0000 Subject: [PATCH 018/194] more --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4523dda2b3..9c55ad2f54 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1607,7 +1607,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; if (pDelFile) { SDelFReader* pDelFReader = NULL; - code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL); + code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); if (code != TSDB_CODE_SUCCESS) { goto _err; } From 43dec55fd740bfa1f5e067f585ccf2455d102ff1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 12:48:58 +0000 Subject: [PATCH 019/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 5 ++-- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 5 ++-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 27 ++++++++++++------- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 60727b5e8e..e7a6f5b8f2 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -244,15 +244,16 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx); int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast, uint8_t **ppBuf); + int8_t cmprAlg, int8_t toLast); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); -int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); +int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); + int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 5ad1f3bcb9..707aa3153d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -760,7 +760,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs if (pDelFile) { SDelFReader *pDelFReader; - code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL); + code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); if (code) goto _err; code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3bb78af44a..4269a785a0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -531,8 +531,7 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { // write block.nSubBlock++; code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock - 1], - ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0, - NULL); + ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0); if (code) goto _err; // put SBlock @@ -569,7 +568,7 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { blockL.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; // write - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1, NULL); + code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1); if (code) goto _err; // push SBlockL diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8ee8b085c0..8528765e74 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -679,6 +679,7 @@ _err: static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; +#if 0 int64_t size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); int64_t n; @@ -729,12 +730,14 @@ static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBloc return code; _err: +#endif return code; } static int32_t tsdbReadColDataImpl(SBlockInfo *pSubBlock, SBlockCol *pBlockCol, SColData *pColData, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; +#if 0 int64_t size; int64_t n; @@ -822,6 +825,7 @@ static int32_t tsdbReadColDataImpl(SBlockInfo *pSubBlock, SBlockCol *pBlockCol, return code; _err: +#endif return code; } @@ -900,10 +904,11 @@ _exit: static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { + int32_t code = 0; +#if 0 TdFilePtr pFD = pReader->pDataFD; SBlockInfo *pSubBlock = &pBlock->aSubBlock[iSubBlock]; SArray *aBlockCol = NULL; - int32_t code = 0; int64_t offset; int64_t size; int64_t n; @@ -1012,6 +1017,7 @@ _exit: _err: taosArrayDestroy(aBlockCol); +#endif return code; } @@ -1071,7 +1077,8 @@ _err: static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlock *pBlock, int32_t iSubBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { - int32_t code = 0; + int32_t code = 0; +#if 0 uint8_t *p; int64_t size; int64_t n; @@ -1148,6 +1155,7 @@ static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlock *pBlock, int32 _err: tsdbError("vgId:%d, tsdb read sub block data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); taosArrayDestroy(aBlockCol); +#endif return code; } @@ -1218,6 +1226,7 @@ _err: int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { int32_t code = 0; +#if 0 tBlockDataReset(pBlockData); @@ -1336,11 +1345,13 @@ int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *p _err: tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); +#endif return code; } int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf) { - int32_t code = 0; + int32_t code = 0; +#if 0 TdFilePtr pFD = pReader->pSmaFD; int64_t offset = pBlock->aSubBlock[0].sOffset; int64_t size = pBlock->aSubBlock[0].nSma * sizeof(SColumnDataAgg) + sizeof(TSCKSUM); @@ -1391,6 +1402,7 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD _err: tsdbError("vgId:%d, read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); tFree(pBuf); +#endif return code; } @@ -1899,14 +1911,12 @@ _err: } int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast, uint8_t **ppBuf) { + int8_t cmprAlg, int8_t toLast) { int32_t code = 0; TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; SDiskData *pDiskData = &pWriter->dData; - uint8_t *pBuf = NULL; - - if (!ppBuf) ppBuf = &pBuf; +#if 0 // convert code = tBlockToDiskData(pBlockData, pDiskData, cmprAlg); if (code) goto _err; @@ -1971,6 +1981,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { } +#endif // ================= SMA ==================== if (pSmaInfo) { @@ -1979,11 +1990,9 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } _exit: - tFree(pBuf); return code; _err: - tFree(pBuf); return code; } From 7c32e099d09b64d9cb689167ecec39ce9e7a493f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 6 Aug 2022 14:20:30 +0000 Subject: [PATCH 020/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 56 +++++++++---------- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index e7a6f5b8f2..2f440842c1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -253,12 +253,12 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); +int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); -int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf); int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9c55ad2f54..87cc000908 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2806,7 +2806,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS SBlockLoadSuppInfo* pSup = &pReader->suppInfo; if (tBlockHasSma(pBlock)) { - code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg, NULL); + code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg); if (code != TSDB_CODE_SUCCESS) { tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), pReader->idStr); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8528765e74..8933aec100 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -829,6 +829,7 @@ _err: return code; } +#if 0 static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SDiskDataHdr *pHdr, SArray *aBlockCol) { int32_t code = 0; int32_t n = 0; @@ -862,6 +863,7 @@ static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SDiskDataHdr _err: return code; } +#endif static int32_t tsdbReadDataArray(uint8_t *pInput, int32_t szInput, int32_t nEle, int8_t type, int8_t cmprAlg, uint8_t **ppOut, uint8_t **ppBuf) { @@ -1349,30 +1351,21 @@ _err: return code; } -int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf) { - int32_t code = 0; -#if 0 - TdFilePtr pFD = pReader->pSmaFD; - int64_t offset = pBlock->aSubBlock[0].sOffset; - int64_t size = pBlock->aSubBlock[0].nSma * sizeof(SColumnDataAgg) + sizeof(TSCKSUM); - uint8_t *pBuf = NULL; - int64_t n; +int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg) { + int32_t code = 0; + SSmaInfo *pSmaInfo = &pBlock->smaInfo; - ASSERT(tBlockHasSma(pBlock)); + ASSERT(pSmaInfo->size > 0); - if (!ppBuf) ppBuf = &pBuf; - code = tRealloc(ppBuf, size); + taosArrayClear(aColumnDataAgg); + + // alloc + int32_t size = pSmaInfo->size + sizeof(TSCKSUM); + code = tRealloc(&pReader->pBuf1, size); if (code) goto _err; - // lseek - n = taosLSeekFile(pFD, offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - // read - n = taosReadFile(pFD, *ppBuf, size); + int64_t n = taosReadFile(pReader->pSmaFD, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1382,27 +1375,27 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD } // check - if (!taosCheckChecksumWhole(*ppBuf, size)) { + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode - taosArrayClear(aColumnDataAgg); - for (int32_t iSma = 0; iSma < pBlock->aSubBlock[0].nSma; iSma++) { - if (taosArrayPush(aColumnDataAgg, &((SColumnDataAgg *)(*ppBuf))[iSma]) == NULL) { + n = 0; + while (n < pSmaInfo->size) { + SColumnDataAgg sma; + + n += tGetColumnDataAgg(pReader->pBuf1 + n, &sma); + if (taosArrayPush(aColumnDataAgg, &sma) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } } - tFree(pBuf); return code; _err: - tsdbError("vgId:%d, read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf); -#endif + tsdbError("vgId:%d tsdb read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } @@ -1912,10 +1905,12 @@ _err: int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast) { - int32_t code = 0; - TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; - SDiskData *pDiskData = &pWriter->dData; + int32_t code = 0; + TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; + ASSERT(pBlockData->nRow > 0); + + // ================= DATA ==================== #if 0 // convert code = tBlockToDiskData(pBlockData, pDiskData, cmprAlg); @@ -1993,6 +1988,7 @@ _exit: return code; _err: + tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } From 85556a3c904ef864b4ecbb9dd084c79415612547 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 06:13:51 +0000 Subject: [PATCH 021/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 38 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 102 ++--- source/dnode/vnode/src/tsdb/tsdbUtil.c | 378 ++++++++---------- 3 files changed, 221 insertions(+), 297 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2f440842c1..fe3992102b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -47,7 +47,6 @@ typedef struct SBlockL SBlockL; typedef struct SColData SColData; typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SBlockData SBlockData; -typedef struct SDiskData SDiskData; typedef struct SDelFile SDelFile; typedef struct SHeadFile SHeadFile; typedef struct SDataFile SDataFile; @@ -152,11 +151,6 @@ SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); -// SDiskData -int32_t tDiskDataInit(SDiskData *pDiskData); -void tDiskDataClear(SDiskData *pDiskData); -int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); -int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); @@ -185,6 +179,9 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg); int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); +int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, + int32_t *szOut, uint8_t **ppBuf); +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); @@ -404,16 +401,15 @@ struct SMapData { }; typedef struct { - int16_t cid; - int8_t type; - int8_t smaOn; - int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE - int32_t szOrigin; // original column value size (only save for variant data type) - int32_t szBitmap; // bitmap size, 0 only for flag == HAS_VAL - int32_t szOffset; // offset size, 0 only for non-variant-length type - int32_t szValue; // value size, 0 when flag == (HAS_NULL | HAS_NONE) - int32_t offset; - uint8_t **ppData; + int16_t cid; + int8_t type; + int8_t smaOn; + int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE + int32_t szOrigin; // original column value size (only save for variant data type) + int32_t szBitmap; // bitmap size, 0 only for flag == HAS_VAL + int32_t szOffset; // offset size, 0 only for non-variant-length type + int32_t szValue; // value size, 0 when flag == (HAS_NULL | HAS_NONE) + int32_t offset; } SBlockCol; struct SBlockInfo { @@ -588,14 +584,6 @@ struct SDelFWriter { uint8_t *pBuf1; }; -struct SDiskData { - SDiskDataHdr hdr; - uint8_t **ppKey; - SArray *aBlockCol; // SArray - int32_t nBuf; - SArray *aBuf; // SArray -}; - struct SDataFWriter { STsdb *pTsdb; SDFileSet wSet; @@ -612,8 +600,6 @@ struct SDataFWriter { uint8_t *pBuf1; uint8_t *pBuf2; - - SDiskData dData; }; struct STsdbReadSnap { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8933aec100..31029e6bb4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1911,72 +1911,58 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock ASSERT(pBlockData->nRow > 0); // ================= DATA ==================== -#if 0 - // convert - code = tBlockToDiskData(pBlockData, pDiskData, cmprAlg); + SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, + .suid = pBlockData->suid, + .uid = pBlockData->uid, + .nRow = pBlockData->nRow, + .cmprAlg = cmprAlg}; + + SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); + if (aBlockCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + // uid + if (pBlockData->uid == 0) { + ASSERT(toLast); + code = tsdbCmprData(); + if (code) goto _err; + } + + // version + code = tsdbCmprData(); if (code) goto _err; - // write the block - if (toLast) { - pBlkInfo->offset = pWriter->fLast.size; - } else { - pBlkInfo->offset = pWriter->fData.size; - } - - // HDR and KEY - int32_t size = tPutDiskDataHdr(NULL, &pDiskData->hdr); - code = tRealloc(ppBuf, size); + // ts + code = tsdbCmprData(); if (code) goto _err; - tPutDiskDataHdr(*ppBuf, &pDiskData->hdr); + // columns + int32_t offset = 0; + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - TSCKSUM cksm = taosCalcChecksum(0, *ppBuf, size); + ASSERT(pColData->flag); - int64_t n = taosWriteFile(pFD, *ppBuf, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (pColData->flag == HAS_NONE) continue; + + SBlockCol blockCol = {.cid = pColData->cid, + .type = pColData->type, + .smaOn = pColData->smaOn, + .flag = pColData->flag, + .szOrigin = pColData->nData}; + + if (pColData->flag != HAS_NULL) { + } + + if (taosArrayPush(aBlockCol, &blockCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } - cksm = taosCalcChecksum(cksm, *pDiskData->ppKey, pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey); - n = taosWriteFile(pFD, *pDiskData->ppKey, pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pFD, &cksm, sizeof(cksm)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pBlkInfo->szKey = size + pDiskData->hdr.szUid + pDiskData->hdr.szVer + pDiskData->hdr.szKey + sizeof(TSCKSUM); - - // SBlockCol - if (pDiskData->hdr.szBlkCol == 0) { - pBlkInfo->szBlock = pBlkInfo->szKey; - goto _write_sma; - } - - code = tRealloc(ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); - if (code) goto _err; - - n = 0; - for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { - n += tPutBlockCol(*ppBuf + n, taosArrayGet(pDiskData->aBlockCol, iBlockCol)); - } - taosCalcChecksumAppend(0, *ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); - - n = taosWriteFile(pFD, *ppBuf, pDiskData->hdr.szBlkCol + sizeof(TSCKSUM)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(pDiskData->aBlockCol); iBlockCol++) { - } -#endif + // write // ================= SMA ==================== if (pSmaInfo) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 0ec922f20e..1b30a17dd2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1509,219 +1509,6 @@ int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData) { return n; } -// SDiskData ============================== -static int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, - int32_t *szOut, uint8_t **ppBuf) { - int32_t code = 0; - - ASSERT(szIn > 0 && ppOut); - - if (cmprAlg == NO_COMPRESSION) { - code = tRealloc(ppOut, nOut + szIn); - if (code) goto _exit; - - memcpy(*ppOut + nOut, pIn, szIn); - *szOut = szIn; - } else { - int32_t size = szIn + COMP_OVERFLOW_BYTES; - - code = tRealloc(ppOut, nOut + size); - if (code) goto _exit; - - if (cmprAlg == TWO_STAGE_COMP) { - ASSERT(ppBuf); - code = tRealloc(ppBuf, size); - if (code) goto _exit; - } - - *szOut = - tDataTypes[type].compFunc(pIn, szIn, szIn / tDataTypes[type].bytes, *ppOut + nOut, size, cmprAlg, *ppBuf, size); - if (*szOut <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _exit; - } - } - -_exit: - return code; -} - -static int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf) { - int32_t code = 0; - - int32_t n = 0; - // bitmap - if (pColData->flag != HAS_VALUE) { - code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, - pBlockCol->ppData, n, &pBlockCol->szBitmap, ppBuf); - if (code) goto _exit; - } else { - pBlockCol->szBitmap = 0; - } - n += pBlockCol->szBitmap; - - // offset - if (IS_VAR_DATA_TYPE(pColData->type)) { - code = tsdbCmprData((uint8_t *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, - pBlockCol->ppData, n, &pBlockCol->szOffset, ppBuf); - if (code) goto _exit; - } else { - pBlockCol->szOffset = 0; - } - n += pBlockCol->szOffset; - - // value - if (pColData->flag != (HAS_NULL | HAS_NONE)) { - code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, pBlockCol->ppData, n, - &pBlockCol->szValue, ppBuf); - if (code) goto _exit; - } else { - pBlockCol->szValue = 0; - } - n += pBlockCol->szValue; - - // checksum - n += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, *ppBuf, n); - -_exit: - return code; -} - -static int32_t tsdbDecmprData() { - int32_t code = 0; - // TODO - return code; -} - -int32_t tDiskDataInit(SDiskData *pDiskData) { - int32_t code = 0; - - pDiskData->aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); - if (pDiskData->aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - pDiskData->aBuf = taosArrayInit(0, sizeof(uint8_t *)); - if (pDiskData->aBuf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - -_exit: - return code; -} - -void tDiskDataClear(SDiskData *pDiskData) { - taosArrayDestroy(pDiskData->aBlockCol); - for (int32_t i = 0; i < taosArrayGetSize(pDiskData->aBuf); i++) { - tFree((uint8_t *)taosArrayGet(pDiskData->aBuf, i)); - } - taosArrayDestroy(pDiskData->aBuf); -} - -static uint8_t **tDiskDataAllocBuf(SDiskData *pDiskData) { - if (pDiskData->nBuf >= taosArrayGetSize(pDiskData->aBuf)) { - uint8_t *p = NULL; - if (taosArrayPush(pDiskData->aBuf, &p) == NULL) { - return NULL; - } - } - - ASSERT(pDiskData->nBuf < taosArrayGetSize(pDiskData->aBuf)); - uint8_t **pp = taosArrayGet(pDiskData->aBuf, pDiskData->nBuf); - pDiskData->nBuf++; - return pp; -} - -int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg) { - int32_t code = 0; - - ASSERT(pBlockData->nRow > 0); - - pDiskData->cmprAlg = cmprAlg; - pDiskData->nRow = pBlockData->nRow; - pDiskData->suid = pBlockData->suid; - pDiskData->uid = pBlockData->uid; - pDiskData->szUid = 0; - pDiskData->szVer = 0; - pDiskData->szKey = 0; - taosArrayClear(pDiskData->aBlockCol); - pDiskData->nBuf = 0; - - { - pDiskData->ppKey = tDiskDataAllocBuf(pDiskData); - if (pDiskData->ppKey == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - int32_t n = 0; - // uid - if (pDiskData->uid == 0) { - code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, pDiskData->ppKey, n, &pDiskData->szUid, NULL); - if (code) goto _exit; - } else { - pDiskData->szUid = 0; - } - n += pDiskData->szUid; - - // version - code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, pDiskData->ppKey, n, &pDiskData->szVer, NULL); - if (code) goto _exit; - n += pDiskData->szVer; - - // tskey - code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, pDiskData->ppKey, &pDiskData->szKey, NULL); - if (code) goto _exit; - } - - // columns - int32_t offset = 0; - for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - if (pColData->flag == HAS_NONE) continue; - - SBlockCol blockCol = {.cid = pColData->cid, - .type = pColData->type, - .smaOn = pColData->smaOn, - .flag = pColData->flag, - .szOrigin = pColData->nData}; - - if (pColData->flag != HAS_NULL) { - // alloc a buffer - blockCol.ppData = tDiskDataAllocBuf(pDiskData); - if (blockCol.ppData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - // compress - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, NULL); - if (code) goto _exit; - - // update offset - blockCol.offset = offset; - offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; - } - - if (taosArrayPush(pDiskData->aBlockCol, &blockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - -_exit: - return code; -} -int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg); -int32_t tDiskToBlockData(SDiskData *pDiskData, SBlockData *pBlockData); - // SDiskDataHdr ============================== int32_t tPutDiskDataHdr(uint8_t *p, void *ph) { int32_t n = 0; @@ -1927,3 +1714,168 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { } } } + +int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, + int32_t *szOut, uint8_t **ppBuf) { + int32_t code = 0; + + ASSERT(szIn > 0 && ppOut); + + if (cmprAlg == NO_COMPRESSION) { + code = tRealloc(ppOut, nOut + szIn); + if (code) goto _exit; + + memcpy(*ppOut + nOut, pIn, szIn); + *szOut = szIn; + } else { + int32_t size = szIn + COMP_OVERFLOW_BYTES; + + code = tRealloc(ppOut, nOut + size); + if (code) goto _exit; + + if (cmprAlg == TWO_STAGE_COMP) { + ASSERT(ppBuf); + code = tRealloc(ppBuf, size); + if (code) goto _exit; + } + + *szOut = + tDataTypes[type].compFunc(pIn, szIn, szIn / tDataTypes[type].bytes, *ppOut + nOut, size, cmprAlg, *ppBuf, size); + if (*szOut <= 0) { + code = TSDB_CODE_COMPRESS_ERROR; + goto _exit; + } + } + +_exit: + return code; +} + +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf) { + int32_t code = 0; + + int32_t n = 0; + // bitmap + if (pColData->flag != HAS_VALUE) { + code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, + pBlockCol->ppData, n, &pBlockCol->szBitmap, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szBitmap = 0; + } + n += pBlockCol->szBitmap; + + // offset + if (IS_VAR_DATA_TYPE(pColData->type)) { + code = tsdbCmprData((uint8_t *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, + pBlockCol->ppData, n, &pBlockCol->szOffset, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szOffset = 0; + } + n += pBlockCol->szOffset; + + // value + if (pColData->flag != (HAS_NULL | HAS_NONE)) { + code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, pBlockCol->ppData, n, + &pBlockCol->szValue, ppBuf); + if (code) goto _exit; + } else { + pBlockCol->szValue = 0; + } + n += pBlockCol->szValue; + + // checksum + n += sizeof(TSCKSUM); + taosCalcChecksumAppend(0, *ppBuf, n); + +_exit: + return code; +} + +#if 0 +int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg) { + int32_t code = 0; + + ASSERT(pBlockData->nRow > 0); + + pDiskData->cmprAlg = cmprAlg; + pDiskData->nRow = pBlockData->nRow; + pDiskData->suid = pBlockData->suid; + pDiskData->uid = pBlockData->uid; + pDiskData->szUid = 0; + pDiskData->szVer = 0; + pDiskData->szKey = 0; + taosArrayClear(pDiskData->aBlockCol); + pDiskData->nBuf = 0; + + { + pDiskData->ppKey = tDiskDataAllocBuf(pDiskData); + if (pDiskData->ppKey == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + int32_t n = 0; + // uid + if (pDiskData->uid == 0) { + code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, pDiskData->ppKey, n, &pDiskData->szUid, NULL); + if (code) goto _exit; + } else { + pDiskData->szUid = 0; + } + n += pDiskData->szUid; + + // version + code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, pDiskData->ppKey, n, &pDiskData->szVer, NULL); + if (code) goto _exit; + n += pDiskData->szVer; + + // tskey + code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, + cmprAlg, pDiskData->ppKey, &pDiskData->szKey, NULL); + if (code) goto _exit; + } + + // columns + int32_t offset = 0; + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + + if (pColData->flag == HAS_NONE) continue; + + SBlockCol blockCol = {.cid = pColData->cid, + .type = pColData->type, + .smaOn = pColData->smaOn, + .flag = pColData->flag, + .szOrigin = pColData->nData}; + + if (pColData->flag != HAS_NULL) { + // alloc a buffer + blockCol.ppData = tDiskDataAllocBuf(pDiskData); + if (blockCol.ppData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + // compress + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, NULL); + if (code) goto _exit; + + // update offset + blockCol.offset = offset; + offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; + } + + if (taosArrayPush(pDiskData->aBlockCol, &blockCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + } + +_exit: + return code; +} +#endif From 2d503135864431cd143859cb60e7d6dec26895be Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 08:12:59 +0000 Subject: [PATCH 022/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 9 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 89 ++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbUtil.c | 43 ++++----- 3 files changed, 95 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index fe3992102b..f746da3857 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -63,6 +63,7 @@ typedef struct SRowMerger SRowMerger; typedef struct STsdbReadSnap STsdbReadSnap; typedef struct SBlockInfo SBlockInfo; typedef struct SSmaInfo SSmaInfo; +typedef struct SBlockCol SBlockCol; #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 @@ -181,7 +182,8 @@ int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, int32_t *szOut, uint8_t **ppBuf); -int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf); +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, + uint8_t **ppBuf); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); @@ -400,7 +402,7 @@ struct SMapData { uint8_t *pData; }; -typedef struct { +struct SBlockCol { int16_t cid; int8_t type; int8_t smaOn; @@ -410,7 +412,7 @@ typedef struct { int32_t szOffset; // offset size, 0 only for non-variant-length type int32_t szValue; // value size, 0 when flag == (HAS_NULL | HAS_NONE) int32_t offset; -} SBlockCol; +}; struct SBlockInfo { int64_t offset; // block data offset @@ -600,6 +602,7 @@ struct SDataFWriter { uint8_t *pBuf1; uint8_t *pBuf2; + uint8_t *pBuf3; }; struct STsdbReadSnap { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 31029e6bb4..416b35aa03 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1414,7 +1414,6 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - code = tDiskDataInit(&pWriter->dData); if (code) goto _err; pWriter->pTsdb = pTsdb; pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, @@ -1594,9 +1593,9 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { goto _err; } - tDiskDataClear(&(*ppWriter)->dData); tFree((*ppWriter)->pBuf1); tFree((*ppWriter)->pBuf2); + tFree((*ppWriter)->pBuf3); taosMemoryFree(*ppWriter); _exit: *ppWriter = NULL; @@ -1905,11 +1904,14 @@ _err: int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast) { - int32_t code = 0; - TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; + int32_t code = 0; ASSERT(pBlockData->nRow > 0); + pBlkInfo->offset = toLast ? pWriter->fLast.size : pWriter->fData.size; + pBlkInfo->szBlock = 0; + pBlkInfo->szKey = 0; + // ================= DATA ==================== SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockData->suid, @@ -1923,23 +1925,8 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock goto _err; } - // uid - if (pBlockData->uid == 0) { - ASSERT(toLast); - code = tsdbCmprData(); - if (code) goto _err; - } - - // version - code = tsdbCmprData(); - if (code) goto _err; - - // ts - code = tsdbCmprData(); - if (code) goto _err; - - // columns - int32_t offset = 0; + // encode ================= + int32_t nBuf1 = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1954,15 +1941,71 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock .szOrigin = pColData->nData}; if (pColData->flag != HAS_NULL) { + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pWriter->pBuf1, nBuf1, &pWriter->pBuf3); + if (code) goto _err; + + blockCol.offset = nBuf1; + nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } if (taosArrayPush(aBlockCol, &blockCol) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + + hdr.szBlkCol += tPutBlockCol(NULL, &blockCol); } - // write + // (uid + version + tskey + aBlockCol) + if (pBlockData->uid == 0) { + code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pWriter->pBuf2, 0, &hdr.szUid, &pWriter->pBuf3); + if (code) goto _err; + } + + code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, &pWriter->pBuf2, hdr.szUid, &hdr.szVer, &pWriter->pBuf3); + if (code) goto _err; + + code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, + cmprAlg, &pWriter->pBuf2, hdr.szUid + hdr.szVer, &hdr.szKey, &pWriter->pBuf3); + if (code) goto _err; + + pBlkInfo->szKey = tPutDiskDataHdr(NULL, &hdr); + code = tRealloc(&pWriter->pBuf3, pBlkInfo->szKey); + if (code) goto _err; + tPutDiskDataHdr(pWriter->pBuf3, &hdr); + TSCKSUM cksm = taosCalcChecksum(0, pWriter->pBuf3, pBlkInfo->szKey); + + // write ================= + TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; + + // hdr + int64_t n = taosWriteFile(pFD, pWriter->pBuf3, pBlkInfo->szKey); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // uid + version + tskey + (CKSM) + taosCalcChecksumAppend(cksm, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + n = taosWriteFile(pFD, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + pBlkInfo->szKey = pBlkInfo->szKey + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM); + + // aBlockCol + + // colmns + + // update info + if (toLast) { + pWriter->fLast.size += pBlkInfo->szBlock; + } else { + pWriter->fData.size += pBlkInfo->szBlock; + } // ================= SMA ==================== if (pSmaInfo) { @@ -1971,10 +2014,12 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } _exit: + taosArrayDestroy(aBlockCol); return code; _err: tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + taosArrayDestroy(aBlockCol); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 1b30a17dd2..8f3bb6e89c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1373,11 +1373,11 @@ int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlock c = tsdbKeyCmprFn(&TSDBROW_KEY(&row1), &TSDBROW_KEY(&row2)); if (c < 0) { - code = tBlockDataAppendRow(pBlockData, &row1, NULL); + // code = tBlockDataAppendRow(pBlockData, &row1, NULL); if (code) goto _exit; iRow1++; } else if (c > 0) { - code = tBlockDataAppendRow(pBlockData, &row2, NULL); + // code = tBlockDataAppendRow(pBlockData, &row2, NULL); if (code) goto _exit; iRow2++; } else { @@ -1387,14 +1387,14 @@ int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlock while (iRow1 < nRow1) { row1 = tsdbRowFromBlockData(pBlockData1, iRow1); - code = tBlockDataAppendRow(pBlockData, &row1, NULL); + // code = tBlockDataAppendRow(pBlockData, &row1, NULL); if (code) goto _exit; iRow1++; } while (iRow2 < nRow2) { row2 = tsdbRowFromBlockData(pBlockData2, iRow2); - code = tBlockDataAppendRow(pBlockData, &row2, NULL); + // code = tBlockDataAppendRow(pBlockData, &row2, NULL); if (code) goto _exit; iRow2++; } @@ -1751,43 +1751,44 @@ _exit: return code; } -int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppBuf) { +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, + uint8_t **ppBuf) { int32_t code = 0; - int32_t n = 0; + pBlockCol->szBitmap = 0; + pBlockCol->szOffset = 0; + pBlockCol->szValue = 0; + + int32_t size = 0; // bitmap if (pColData->flag != HAS_VALUE) { - code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, - pBlockCol->ppData, n, &pBlockCol->szBitmap, ppBuf); + code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, + nOut + size, &pBlockCol->szBitmap, ppBuf); if (code) goto _exit; - } else { - pBlockCol->szBitmap = 0; } - n += pBlockCol->szBitmap; + size += pBlockCol->szBitmap; // offset if (IS_VAR_DATA_TYPE(pColData->type)) { code = tsdbCmprData((uint8_t *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg, - pBlockCol->ppData, n, &pBlockCol->szOffset, ppBuf); + ppOut, nOut + size, &pBlockCol->szOffset, ppBuf); if (code) goto _exit; - } else { - pBlockCol->szOffset = 0; } - n += pBlockCol->szOffset; + size += pBlockCol->szOffset; // value if (pColData->flag != (HAS_NULL | HAS_NONE)) { - code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, pBlockCol->ppData, n, + code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, ppOut, nOut + size, &pBlockCol->szValue, ppBuf); if (code) goto _exit; - } else { - pBlockCol->szValue = 0; } - n += pBlockCol->szValue; + size += pBlockCol->szValue; // checksum - n += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, *ppBuf, n); + size += sizeof(TSCKSUM); + code = tRealloc(ppOut, nOut + size); + if (code) goto _exit; + taosCalcChecksumAppend(0, *ppOut + nOut, size); _exit: return code; From 94069ec20a1c20328986d888cc052ded5378d795 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 08:35:57 +0000 Subject: [PATCH 023/194] more work --- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 38 +++++++- source/dnode/vnode/src/tsdb/tsdbUtil.c | 87 ------------------- 2 files changed, 36 insertions(+), 89 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 416b35aa03..1d5625220d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1926,6 +1926,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } // encode ================= + // columns int32_t nBuf1 = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1956,27 +1957,34 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock hdr.szBlkCol += tPutBlockCol(NULL, &blockCol); } - // (uid + version + tskey + aBlockCol) + // uid if (pBlockData->uid == 0) { code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, &pWriter->pBuf2, 0, &hdr.szUid, &pWriter->pBuf3); if (code) goto _err; } + // version code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, &pWriter->pBuf2, hdr.szUid, &hdr.szVer, &pWriter->pBuf3); if (code) goto _err; + // tskey code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg, &pWriter->pBuf2, hdr.szUid + hdr.szVer, &hdr.szKey, &pWriter->pBuf3); if (code) goto _err; + // hdr pBlkInfo->szKey = tPutDiskDataHdr(NULL, &hdr); code = tRealloc(&pWriter->pBuf3, pBlkInfo->szKey); if (code) goto _err; tPutDiskDataHdr(pWriter->pBuf3, &hdr); TSCKSUM cksm = taosCalcChecksum(0, pWriter->pBuf3, pBlkInfo->szKey); + code = tRealloc(&pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + if (code) goto _err; + taosCalcChecksumAppend(cksm, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + // write ================= TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; @@ -1988,17 +1996,43 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } // uid + version + tskey + (CKSM) - taosCalcChecksumAppend(cksm, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); n = taosWriteFile(pFD, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } pBlkInfo->szKey = pBlkInfo->szKey + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM); + pBlkInfo->szBlock += pBlkInfo->szKey; // aBlockCol + if (hdr.szBlkCol > 0) { + code = tRealloc(&pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); + if (code) goto _err; + + n = 0; + for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(aBlockCol); iBlockCol++) { + n += tPutBlockCol(pWriter->pBuf2 + n, taosArrayGet(aBlockCol, iBlockCol)); + } + ASSERT(n == hdr.szBlkCol); + taosCalcChecksumAppend(0, pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); + + n = taosWriteFile(pFD, pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } // colmns + if (nBuf1 > 0) { + n = taosWriteFile(pFD, pWriter->pBuf1, nBuf1); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pBlkInfo->szBlock += nBuf1; + } // update info if (toLast) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8f3bb6e89c..d669c28960 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1793,90 +1793,3 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol _exit: return code; } - -#if 0 -int32_t tBlockToDiskData(SBlockData *pBlockData, SDiskData *pDiskData, int8_t cmprAlg) { - int32_t code = 0; - - ASSERT(pBlockData->nRow > 0); - - pDiskData->cmprAlg = cmprAlg; - pDiskData->nRow = pBlockData->nRow; - pDiskData->suid = pBlockData->suid; - pDiskData->uid = pBlockData->uid; - pDiskData->szUid = 0; - pDiskData->szVer = 0; - pDiskData->szKey = 0; - taosArrayClear(pDiskData->aBlockCol); - pDiskData->nBuf = 0; - - { - pDiskData->ppKey = tDiskDataAllocBuf(pDiskData); - if (pDiskData->ppKey == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - int32_t n = 0; - // uid - if (pDiskData->uid == 0) { - code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, pDiskData->ppKey, n, &pDiskData->szUid, NULL); - if (code) goto _exit; - } else { - pDiskData->szUid = 0; - } - n += pDiskData->szUid; - - // version - code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, pDiskData->ppKey, n, &pDiskData->szVer, NULL); - if (code) goto _exit; - n += pDiskData->szVer; - - // tskey - code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, pDiskData->ppKey, &pDiskData->szKey, NULL); - if (code) goto _exit; - } - - // columns - int32_t offset = 0; - for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - if (pColData->flag == HAS_NONE) continue; - - SBlockCol blockCol = {.cid = pColData->cid, - .type = pColData->type, - .smaOn = pColData->smaOn, - .flag = pColData->flag, - .szOrigin = pColData->nData}; - - if (pColData->flag != HAS_NULL) { - // alloc a buffer - blockCol.ppData = tDiskDataAllocBuf(pDiskData); - if (blockCol.ppData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - // compress - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, NULL); - if (code) goto _exit; - - // update offset - blockCol.offset = offset; - offset = offset + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; - } - - if (taosArrayPush(pDiskData->aBlockCol, &blockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - -_exit: - return code; -} -#endif From 3698016ad9ee5d62ec7cbfabdb3e76970cb6957b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 09:25:50 +0000 Subject: [PATCH 024/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 1 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 91 ++++++++----------- 2 files changed, 40 insertions(+), 52 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f746da3857..5bbc0d45f1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -603,6 +603,7 @@ struct SDataFWriter { uint8_t *pBuf1; uint8_t *pBuf2; uint8_t *pBuf3; + uint8_t *pBuf4; }; struct STsdbReadSnap { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 1d5625220d..faabf60f71 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1596,6 +1596,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { tFree((*ppWriter)->pBuf1); tFree((*ppWriter)->pBuf2); tFree((*ppWriter)->pBuf3); + tFree((*ppWriter)->pBuf4); taosMemoryFree(*ppWriter); _exit: *ppWriter = NULL; @@ -1919,14 +1920,8 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock .nRow = pBlockData->nRow, .cmprAlg = cmprAlg}; - SArray *aBlockCol = taosArrayInit(taosArrayGetSize(pBlockData->aIdx), sizeof(SBlockCol)); - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - // encode ================= - // columns + // columns AND SBlockCol int32_t nBuf1 = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1949,89 +1944,83 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } - if (taosArrayPush(aBlockCol, &blockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - hdr.szBlkCol += tPutBlockCol(NULL, &blockCol); + code = tRealloc(&pWriter->pBuf2, hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); + if (code) goto _err; + hdr.szBlkCol += tPutBlockCol(pWriter->pBuf2 + hdr.szBlkCol, &blockCol); } - // uid + int32_t nBuf2 = 0; + if (hdr.szBlkCol > 0) { + nBuf2 = hdr.szBlkCol + sizeof(TSCKSUM); + + code = tRealloc(&pWriter->pBuf2, nBuf2); + if (code) goto _err; + + taosCalcChecksumAppend(0, pWriter->pBuf2, nBuf2); + } + + // uid + version + tskey + int32_t nBuf3 = 0; if (pBlockData->uid == 0) { code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pWriter->pBuf2, 0, &hdr.szUid, &pWriter->pBuf3); + &pWriter->pBuf3, nBuf3, &hdr.szUid, &pWriter->pBuf4); if (code) goto _err; } + nBuf3 += hdr.szUid; - // version code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &pWriter->pBuf2, hdr.szUid, &hdr.szVer, &pWriter->pBuf3); + cmprAlg, &pWriter->pBuf3, nBuf3, &hdr.szVer, &pWriter->pBuf4); if (code) goto _err; + nBuf3 += hdr.szVer; - // tskey code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &pWriter->pBuf2, hdr.szUid + hdr.szVer, &hdr.szKey, &pWriter->pBuf3); + cmprAlg, &pWriter->pBuf3, nBuf3, &hdr.szKey, &pWriter->pBuf4); + if (code) goto _err; + nBuf3 += hdr.szKey; + + nBuf3 += sizeof(TSCKSUM); + code = tRealloc(&pWriter->pBuf3, nBuf3); if (code) goto _err; // hdr - pBlkInfo->szKey = tPutDiskDataHdr(NULL, &hdr); - code = tRealloc(&pWriter->pBuf3, pBlkInfo->szKey); + int32_t nBuf4 = tPutDiskDataHdr(NULL, &hdr); + code = tRealloc(&pWriter->pBuf4, nBuf4); if (code) goto _err; - tPutDiskDataHdr(pWriter->pBuf3, &hdr); - TSCKSUM cksm = taosCalcChecksum(0, pWriter->pBuf3, pBlkInfo->szKey); - - code = tRealloc(&pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); - if (code) goto _err; - taosCalcChecksumAppend(cksm, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + tPutDiskDataHdr(pWriter->pBuf4, &hdr); + taosCalcChecksumAppend(taosCalcChecksum(0, pWriter->pBuf4, nBuf4), pWriter->pBuf3, nBuf3); // write ================= TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; - // hdr - int64_t n = taosWriteFile(pFD, pWriter->pBuf3, pBlkInfo->szKey); + pBlkInfo->szKey = nBuf4 + nBuf3; + pBlkInfo->szBlock = nBuf1 + nBuf2 + nBuf3 + nBuf4; + + int64_t n = taosWriteFile(pFD, pWriter->pBuf4, nBuf4); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - // uid + version + tskey + (CKSM) - n = taosWriteFile(pFD, pWriter->pBuf2, hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM)); + n = taosWriteFile(pFD, pWriter->pBuf3, nBuf3); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - pBlkInfo->szKey = pBlkInfo->szKey + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM); - pBlkInfo->szBlock += pBlkInfo->szKey; - // aBlockCol - if (hdr.szBlkCol > 0) { - code = tRealloc(&pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); - if (code) goto _err; - - n = 0; - for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(aBlockCol); iBlockCol++) { - n += tPutBlockCol(pWriter->pBuf2 + n, taosArrayGet(aBlockCol, iBlockCol)); - } - ASSERT(n == hdr.szBlkCol); - taosCalcChecksumAppend(0, pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); - - n = taosWriteFile(pFD, pWriter->pBuf2, hdr.szBlkCol + sizeof(TSCKSUM)); + if (nBuf2) { + n = taosWriteFile(pFD, pWriter->pBuf2, nBuf2); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } } - // colmns - if (nBuf1 > 0) { + if (nBuf1) { n = taosWriteFile(pFD, pWriter->pBuf1, nBuf1); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - - pBlkInfo->szBlock += nBuf1; } // update info @@ -2048,12 +2037,10 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } _exit: - taosArrayDestroy(aBlockCol); return code; _err: tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - taosArrayDestroy(aBlockCol); return code; } From 605e89b6a17156d82642b1c071e467213b750760 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 12:50:42 +0000 Subject: [PATCH 025/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 6 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 202 +++++++++++++----- source/dnode/vnode/src/tsdb/tsdbUtil.c | 75 +++++++ 3 files changed, 235 insertions(+), 48 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 5bbc0d45f1..a824f8f0b3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -182,8 +182,12 @@ int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, int32_t *szOut, uint8_t **ppBuf); +int32_t tsdbDecmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t szOut, + uint8_t **ppBuf); int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, uint8_t **ppBuf); +int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, + uint8_t **ppBuf); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); @@ -254,6 +258,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMa int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); +#if 0 int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, @@ -262,6 +267,7 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl uint8_t **ppBuf2); int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); +#endif // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index faabf60f71..125e1bcb73 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -677,6 +677,159 @@ _err: return code; } +int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg) { + int32_t code = 0; + SSmaInfo *pSmaInfo = &pBlock->smaInfo; + + ASSERT(pSmaInfo->size > 0); + + taosArrayClear(aColumnDataAgg); + + // alloc + int32_t size = pSmaInfo->size + sizeof(TSCKSUM); + code = tRealloc(&pReader->pBuf1, size); + if (code) goto _err; + + // read + int64_t n = taosReadFile(pReader->pSmaFD, pReader->pBuf1, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n < size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // check + if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // decode + n = 0; + while (n < pSmaInfo->size) { + SColumnDataAgg sma; + + n += tGetColumnDataAgg(pReader->pBuf1 + n, &sma); + if (taosArrayPush(aColumnDataAgg, &sma) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + + return code; + +_err: + tsdbError("vgId:%d tsdb read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, int8_t fromLast, int16_t *aColId, + int32_t nColId, SBlockData *pBlockData) { + int32_t code = 0; + + tBlockDataReset(pBlockData); + + TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; + + // seek + int64_t n = taosLSeekFile(pFD, pBlkInfo->offset, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // read + code = tRealloc(&pReader->pBuf1, pBlkInfo->szBlock); + if (code) goto _err; + + n = taosReadFile(pFD, pReader->pBuf1, pBlkInfo->szBlock); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n < pBlkInfo->szBlock) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + uint8_t *p = pReader->pBuf1; + // check & decode + SDiskDataHdr hdr; + if (!taosCheckChecksumWhole(p, pBlkInfo->szKey)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + p += tGetDiskDataHdr(p, &hdr); + + tBlockDataSetSchema(pBlockData, NULL, hdr.suid, hdr.uid); + pBlockData->nRow = hdr.nRow; + + if (hdr.uid == 0) { + ASSERT(hdr.szUid); + code = tsdbDecmprData(p, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aUid, + sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); + if (code) goto _err; + } else { + ASSERT(hdr.szUid == 0); + } + p += hdr.szUid; + + code = tsdbDecmprData(p, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aVersion, + sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); + if (code) goto _err; + p += hdr.szVer; + + code = tsdbDecmprData(p, hdr.szKey, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, + sizeof(TSKEY) * hdr.nRow, &pReader->pBuf2); + if (code) goto _err; + p += hdr.szKey; + p += sizeof(TSCKSUM); + + // SBlockCol + if (hdr.szBlkCol > 0) { + if (!taosCheckChecksumWhole(p, hdr.szBlkCol + sizeof(TSCKSUM))) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + int32_t iColData = 0; + uint8_t *pt = p + hdr.szBlkCol + sizeof(TSCKSUM); + n = 0; + while (n < hdr.szBlkCol) { + SBlockCol blockCol; + + n += tGetBlockCol(p + n, &blockCol); + + ASSERT(blockCol.flag && blockCol.flag != HAS_NONE); + + SColData *pColData; + code = tBlockDataAddColData(pBlockData, iColData, &pColData); + if (code) goto _err; + iColData++; + + tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); + + if (blockCol.flag == HAS_NULL) { + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type)); + if (code) goto _err; + } + } else { + code = tsdbDecmprColData(pt + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, pReader->pBuf2); + if (code) goto _err; + } + } + } + + return code; + +_err: + tsdbError("vgId:%d tsdb read block data impl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + +#if 0 static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; #if 0 @@ -1350,54 +1503,7 @@ _err: #endif return code; } - -int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg) { - int32_t code = 0; - SSmaInfo *pSmaInfo = &pBlock->smaInfo; - - ASSERT(pSmaInfo->size > 0); - - taosArrayClear(aColumnDataAgg); - - // alloc - int32_t size = pSmaInfo->size + sizeof(TSCKSUM); - code = tRealloc(&pReader->pBuf1, size); - if (code) goto _err; - - // read - int64_t n = taosReadFile(pReader->pSmaFD, pReader->pBuf1, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // decode - n = 0; - while (n < pSmaInfo->size) { - SColumnDataAgg sma; - - n += tGetColumnDataAgg(pReader->pBuf1 + n, &sma); - if (taosArrayPush(aColumnDataAgg, &sma) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - - return code; - -_err: - tsdbError("vgId:%d tsdb read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - return code; -} +#endif // SDataFWriter ==================================================== int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d669c28960..dbb35532e8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1751,6 +1751,36 @@ _exit: return code; } +int32_t tsdbDecmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t szOut, + uint8_t **ppBuf) { + int32_t code = 0; + + code = tRealloc(ppOut, szOut); + if (code) goto _exit; + + if (cmprAlg == NO_COMPRESSION) { + ASSERT(szIn == szOut); + memcpy(*ppOut, pIn, szOut); + } else { + if (cmprAlg == TWO_STAGE_COMP) { + code = tRealloc(ppBuf, szOut + COMP_OVERFLOW_BYTES); + if (code) goto _exit; + } + + int32_t size = tDataTypes[type].decompFunc(pIn, szIn, szOut / tDataTypes[type].bytes, *ppOut, szOut, cmprAlg, + *ppBuf, szOut + COMP_OVERFLOW_BYTES); + if (size <= 0) { + code = TSDB_CODE_COMPRESS_ERROR; + goto _exit; + } + + ASSERT(size == szOut); + } + +_exit: + return code; +} + int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, uint8_t **ppBuf) { int32_t code = 0; @@ -1793,3 +1823,48 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol _exit: return code; } + +int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, + uint8_t **ppBuf) { + int32_t code = 0; + + int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); + if (!taosCheckChecksumWhole(pIn, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + + pColData->cid = pBlockCol->cid; + pColData->type = pBlockCol->type; + pColData->smaOn = pBlockCol->smaOn; + pColData->flag = pBlockCol->flag; + pColData->nVal = nVal; + pColData->nData = pBlockCol->szOrigin; + + uint8_t *p = pIn; + // bitmap + if (pBlockCol->szBitmap) { + code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, + BIT2_SIZE(pColData->nVal), ppBuf); + if (code) goto _exit; + } + p += pBlockCol->szBitmap; + + // offset + if (pBlockCol->szOffset) { + code = tsdbDecmprData(p, pBlockCol->szOffset, TSDB_DATA_TYPE_INT, cmprAlg, (uint8_t **)&pColData->aOffset, + sizeof(int32_t) * pColData->nVal, ppBuf); + if (code) goto _exit; + } + p += pBlockCol->szOffset; + + // value + if (pBlockCol->szValue) { + code = tsdbDecmprData(p, pBlockCol->szValue, pColData->type, cmprAlg, &pColData->pData, pColData->nData, ppBuf); + if (code) goto _exit; + } + p += pBlockCol->szValue; + +_exit: + return code; +} \ No newline at end of file From 71691eb7daaa38bd9a3852444ed5103c66da326f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 13:14:43 +0000 Subject: [PATCH 026/194] make code compile --- source/dnode/vnode/src/inc/tsdb.h | 4 +++ source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 +-- source/dnode/vnode/src/tsdb/tsdbRead.c | 3 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 35 ++++++++++++++++++- 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a824f8f0b3..6af1a1a79b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -257,6 +257,10 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, int16_t *aColId, + int32_t nColId); +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, int16_t *aColId, + int32_t nColId); #if 0 int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 707aa3153d..c2bfad264b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -507,7 +507,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetBlock); /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ - code = tsdbReadBlockData(state->pDataFReader, state->pBlockIdx, &block, state->pBlockData, NULL, NULL); + code = tsdbReadDataBlock(state->pDataFReader, &block, state->pBlockData, NULL, 0); if (code) goto _err; state->nRow = state->blockData.nRow; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 4269a785a0..ad721ede7a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -309,7 +309,7 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { pCommitter->dReader.iBlockL++; if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal, NULL, NULL); + code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal, NULL, 0); if (code) goto _exit; pCommitter->dReader.iRow = 0; @@ -593,7 +593,7 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S SBlockData *pBlockDataR = &pCommitter->dReader.bData; SBlockData *pBlockDataW = &pCommitter->dWriter.bData; - code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR, NULL, NULL); + code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR, NULL, 0); if (code) goto _err; tBlockDataClearData(pBlockDataW); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 87cc000908..0157f0ed22 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -785,8 +785,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockIdx blockIdx = {.suid = pReader->suid, .uid = pBlockScanInfo->uid}; - int32_t code = - tsdbReadColData(pReader->pFileReader, &blockIdx, pBlock, pSupInfo->colIds, numOfCols, pBlockData, NULL, NULL); + int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData, pSupInfo->colIds, numOfCols); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 125e1bcb73..4f02299c7e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -816,7 +816,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (code) goto _err; } } else { - code = tsdbDecmprColData(pt + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, pReader->pBuf2); + code = tsdbDecmprColData(pt + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf2); if (code) goto _err; } } @@ -829,6 +829,39 @@ _err: return code; } +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, int16_t *aColId, + int32_t nColId) { + int32_t code = 0; + + code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, aColId, nColId, pBlockData); + if (code) goto _err; + + for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { + // TODO + ASSERT(0); + } + + return code; + +_err: + tsdbError("vgId:%d tsdb read data block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, int16_t *aColId, + int32_t nColId) { + int32_t code = 0; + + code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, aColId, nColId, pBlockData); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + #if 0 static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; From 7671bfe19e03d87acfcc67be7dfcfad3056cef45 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 7 Aug 2022 15:20:28 +0000 Subject: [PATCH 027/194] more refact --- source/dnode/vnode/src/inc/tsdb.h | 1 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 81 ++++++++----------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 34 ++++++++ 3 files changed, 67 insertions(+), 49 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 6af1a1a79b..e1bbc3278d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -188,6 +188,7 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol uint8_t **ppBuf); int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, uint8_t **ppBuf); +int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 4f02299c7e..94d957eac7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -401,6 +401,7 @@ struct SDataFReader { uint8_t *pBuf1; uint8_t *pBuf2; + uint8_t *pBuf3; }; int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { @@ -485,6 +486,7 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { tFree((*ppReader)->pBuf1); tFree((*ppReader)->pBuf2); + tFree((*ppReader)->pBuf3); taosMemoryFree(*ppReader); _exit: @@ -729,94 +731,75 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo int32_t nColId, SBlockData *pBlockData) { int32_t code = 0; + // TODO tBlockDataReset(pBlockData); TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; - // seek - int64_t n = taosLSeekFile(pFD, pBlkInfo->offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // read - code = tRealloc(&pReader->pBuf1, pBlkInfo->szBlock); + // uid + version + tskey + code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey); if (code) goto _err; - - n = taosReadFile(pFD, pReader->pBuf1, pBlkInfo->szBlock); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < pBlkInfo->szBlock) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - uint8_t *p = pReader->pBuf1; - // check & decode SDiskDataHdr hdr; - if (!taosCheckChecksumWhole(p, pBlkInfo->szKey)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - p += tGetDiskDataHdr(p, &hdr); + uint8_t *p = pReader->pBuf1 + tGetDiskDataHdr(pReader->pBuf1, &hdr); - tBlockDataSetSchema(pBlockData, NULL, hdr.suid, hdr.uid); + ASSERT(hdr.delimiter == TSDB_FILE_DLMT); + ASSERT(hdr.suid || hdr.uid); + + pBlockData->suid = hdr.suid; + pBlockData->uid = hdr.uid; pBlockData->nRow = hdr.nRow; + // uid if (hdr.uid == 0) { ASSERT(hdr.szUid); code = tsdbDecmprData(p, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aUid, sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); if (code) goto _err; } else { - ASSERT(hdr.szUid == 0); + ASSERT(!hdr.szUid); } p += hdr.szUid; + // version code = tsdbDecmprData(p, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aVersion, sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); if (code) goto _err; p += hdr.szVer; - code = tsdbDecmprData(p, hdr.szKey, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, + // TSKEY + code = tsdbDecmprData(p, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * hdr.nRow, &pReader->pBuf2); if (code) goto _err; p += hdr.szKey; - p += sizeof(TSCKSUM); - // SBlockCol + ASSERT(p - pReader->pBuf1 == sizeof(TSCKSUM)); + + // read and decode columns if (hdr.szBlkCol > 0) { - if (!taosCheckChecksumWhole(p, hdr.szBlkCol + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + code = + tsdbReadAndCheckFile(pFD, pBlkInfo->offset + pBlkInfo->szKey, &pReader->pBuf1, hdr.szBlkCol + sizeof(TSCKSUM)); + if (code) goto _err; - int32_t iColData = 0; - uint8_t *pt = p + hdr.szBlkCol + sizeof(TSCKSUM); - n = 0; + int32_t n = 0; while (n < hdr.szBlkCol) { SBlockCol blockCol; - n += tGetBlockCol(p + n, &blockCol); + n += tGetBlockCol(pReader->pBuf1 + n, &blockCol); ASSERT(blockCol.flag && blockCol.flag != HAS_NONE); - SColData *pColData; - code = tBlockDataAddColData(pBlockData, iColData, &pColData); - if (code) goto _err; - iColData++; + // TODO: merge with the column IDs - tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); + SColData *pColData = NULL; // (todo) if (blockCol.flag == HAS_NULL) { - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type)); - if (code) goto _err; - } + // TODO: make a hdr.nRow COL_VAL_NULL(); } else { - code = tsdbDecmprColData(pt + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf2); + code = tsdbReadAndCheckFile( + pFD, pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &pReader->pBuf2, + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM)); + + code = tsdbDecmprColData(pReader->pBuf2, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf3); if (code) goto _err; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index dbb35532e8..6a27b10cf1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1865,6 +1865,40 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in } p += pBlockCol->szValue; +_exit: + return code; +} + +int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size) { + int32_t code = 0; + + // alloc + code = tRealloc(ppOut, size); + if (code) goto _exit; + + // seek + int64_t n = taosLSeekFile(pFD, offset, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + + // read + n = taosReadFile(pFD, *ppOut, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } else if (n < size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + + // check + if (!taosCheckChecksumWhole(*ppOut, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + _exit: return code; } \ No newline at end of file From 33c3f34f3489732a57ce92c7a26abe3c6a5f7e55 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 03:22:24 +0000 Subject: [PATCH 028/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 17 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 14 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 16 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 167 ++---------------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 14 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 4 +- 7 files changed, 49 insertions(+), 189 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index e1bbc3278d..f33b8cd51d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -137,8 +137,10 @@ int32_t tGetColData(uint8_t *p, SColData *pColData); #define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) #define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) #define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) -int32_t tBlockDataInit(SBlockData *pBlockData); -void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear); + +int32_t tBlockDataCreate(SBlockData *pBlockData); +void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); + void tBlockDataReset(SBlockData *pBlockData); int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid); int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); @@ -262,17 +264,6 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl int32_t nColId); int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, int16_t *aColId, int32_t nColId); - -#if 0 -int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, - SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2); -int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData, - uint8_t **ppBuf1, uint8_t **ppBuf2); -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2); -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2); -#endif // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index c2bfad264b..2cb3f1f46d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -450,9 +450,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { - // tBlockDataClear(&state->blockData, 1); + // tBlockDataDestroy(&state->blockData, 1); if (state->pBlockData) { - tBlockDataClear(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData, 1); state->pBlockData = NULL; } @@ -494,7 +494,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (!state->pBlockData) { state->pBlockData = &state->blockData; - tBlockDataInit(&state->blockData); + tBlockDataCreate(&state->blockData); } } case SFSNEXTROW_BLOCKDATA: @@ -552,8 +552,8 @@ _err: state->aBlockIdx = NULL; } if (state->pBlockData) { - // tBlockDataClear(&state->blockData, 1); - tBlockDataClear(state->pBlockData, 1); + // tBlockDataDestroy(&state->blockData, 1); + tBlockDataDestroy(state->pBlockData, 1); state->pBlockData = NULL; } @@ -579,8 +579,8 @@ int32_t clearNextRowFromFS(void *iter) { state->aBlockIdx = NULL; } if (state->pBlockData) { - // tBlockDataClear(&state->blockData, 1); - tBlockDataClear(state->pBlockData, 1); + // tBlockDataDestroy(&state->blockData, 1); + tBlockDataDestroy(state->pBlockData, 1); state->pBlockData = NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index ad721ede7a..44f360d597 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1270,7 +1270,7 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { goto _exit; } - code = tBlockDataInit(&pCommitter->dReader.bData); + code = tBlockDataCreate(&pCommitter->dReader.bData); if (code) goto _exit; pCommitter->dReader.aBlockL = taosArrayInit(0, sizeof(SBlockL)); @@ -1279,7 +1279,7 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { goto _exit; } - code = tBlockDataInit(&pCommitter->dReader.bDatal); + code = tBlockDataCreate(&pCommitter->dReader.bDatal); if (code) goto _exit; // Writer @@ -1295,10 +1295,10 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { goto _exit; } - code = tBlockDataInit(&pCommitter->dWriter.bData); + code = tBlockDataCreate(&pCommitter->dWriter.bData); if (code) goto _exit; - code = tBlockDataInit(&pCommitter->dWriter.bDatal); + code = tBlockDataCreate(&pCommitter->dWriter.bDatal); if (code) goto _exit; _exit: @@ -1309,16 +1309,16 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { // Reader taosArrayDestroy(pCommitter->dReader.aBlockIdx); tMapDataClear(&pCommitter->dReader.mBlock); - tBlockDataClear(&pCommitter->dReader.bData, 1); + tBlockDataDestroy(&pCommitter->dReader.bData, 1); taosArrayDestroy(pCommitter->dReader.aBlockL); - tBlockDataClear(&pCommitter->dReader.bDatal, 1); + tBlockDataDestroy(&pCommitter->dReader.bDatal, 1); // Writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); taosArrayDestroy(pCommitter->dWriter.aBlockL); tMapDataClear(&pCommitter->dWriter.mBlock); - tBlockDataClear(&pCommitter->dWriter.bData, 1); - tBlockDataClear(&pCommitter->dWriter.bDatal, 1); + tBlockDataDestroy(&pCommitter->dWriter.bData, 1); + tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0157f0ed22..cd1e46c342 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -438,7 +438,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; - code = tBlockDataInit(&pReader->status.fileBlockData); + code = tBlockDataCreate(&pReader->status.fileBlockData); if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _end; @@ -2670,7 +2670,7 @@ void tsdbReaderClose(STsdbReader* pReader) { } } taosMemoryFree(pSupInfo->buildBuf); - tBlockDataClear(&pReader->status.fileBlockData, true); + tBlockDataDestroy(&pReader->status.fileBlockData, true); cleanupDataBlockIterator(&pReader->status.blockIter); @@ -2874,7 +2874,7 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { tBlockDataClearData(&pStatus->fileBlockData); int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, pBlockScanInfo, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { - tBlockDataClear(&pStatus->fileBlockData, 1); + tBlockDataDestroy(&pStatus->fileBlockData, 1); terrno = code; return NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 94d957eac7..69f39bb3c5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -846,63 +846,6 @@ _err: } #if 0 -static int32_t tsdbReadBlockDataKey(SBlockData *pBlockData, SBlockInfo *pSubBlock, uint8_t *pBuf, uint8_t **ppBuf) { - int32_t code = 0; -#if 0 - int64_t size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); - int64_t n; - - if (!taosCheckChecksumWhole(pBuf, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - code = tRealloc((uint8_t **)&pBlockData->aVersion, sizeof(int64_t) * pSubBlock->nRow); - if (code) goto _err; - code = tRealloc((uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * pSubBlock->nRow); - if (code) goto _err; - - if (pSubBlock->cmprAlg == NO_COMPRESSION) { - ASSERT(pSubBlock->szVersion == sizeof(int64_t) * pSubBlock->nRow); - ASSERT(pSubBlock->szTSKEY == sizeof(TSKEY) * pSubBlock->nRow); - - // VERSION - memcpy(pBlockData->aVersion, pBuf, pSubBlock->szVersion); - - // TSKEY - memcpy(pBlockData->aTSKEY, pBuf + pSubBlock->szVersion, pSubBlock->szTSKEY); - } else { - size = sizeof(int64_t) * pSubBlock->nRow + COMP_OVERFLOW_BYTES; - if (pSubBlock->cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf, size); - if (code) goto _err; - } - - // VERSION - n = tsDecompressBigint(pBuf, pSubBlock->szVersion, pSubBlock->nRow, (char *)pBlockData->aVersion, - sizeof(int64_t) * pSubBlock->nRow, pSubBlock->cmprAlg, *ppBuf, size); - if (n < 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - // TSKEY - n = tsDecompressTimestamp(pBuf + pSubBlock->szVersion, pSubBlock->szTSKEY, pSubBlock->nRow, - (char *)pBlockData->aTSKEY, sizeof(TSKEY) * pSubBlock->nRow, pSubBlock->cmprAlg, *ppBuf, - size); - if (n < 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - } - - return code; - -_err: -#endif - return code; -} - static int32_t tsdbReadColDataImpl(SBlockInfo *pSubBlock, SBlockCol *pBlockCol, SColData *pColData, uint8_t *pBuf, uint8_t **ppBuf) { int32_t code = 0; @@ -998,80 +941,6 @@ _err: return code; } -#if 0 -static int32_t tsdbReadBlockCol(uint8_t *pBuf, int32_t szBlockCol, SDiskDataHdr *pHdr, SArray *aBlockCol) { - int32_t code = 0; - int32_t n = 0; - SBlockCol blockCol; - SBlockCol *pBlockCol = &blockCol; - - // checksum - if (!taosCheckChecksumWhole(pBuf, szBlockCol + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // hdr - *pHdr = *(SDiskDataHdr *)pBuf; - n += sizeof(SDiskDataHdr); - - // aBlockCol - while (n < szBlockCol) { - n += tGetBlockCol(pBuf + n, pBlockCol); - - if (taosArrayPush(aBlockCol, pBlockCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - - ASSERT(n == szBlockCol); - - return code; - -_err: - return code; -} -#endif - -static int32_t tsdbReadDataArray(uint8_t *pInput, int32_t szInput, int32_t nEle, int8_t type, int8_t cmprAlg, - uint8_t **ppOut, uint8_t **ppBuf) { - int32_t code = 0; - int32_t size; - - // size - if (IS_VAR_DATA_TYPE(type)) { - size = nEle; - } else { - size = tDataTypes[type].bytes * nEle; - } - - // alloc - code = tRealloc(ppOut, size); - if (code) goto _exit; - - // decode - if (cmprAlg == NO_COMPRESSION) { - ASSERT(szInput == size); - memcpy(*ppOut, pInput, size); - } else { - if (cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf, size + COMP_OVERFLOW_BYTES); - if (code) goto _exit; - - int32_t n = - tDataTypes[type].decompFunc(pInput, szInput, nEle, *ppOut, size, cmprAlg, *ppBuf, size + COMP_OVERFLOW_BYTES); - if (n <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _exit; - } - } - } - -_exit: - return code; -} - static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { @@ -1210,29 +1079,29 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl SBlockData *pBlockData1 = &(SBlockData){0}; SBlockData *pBlockData2 = &(SBlockData){0}; - tBlockDataInit(pBlockData1); - tBlockDataInit(pBlockData2); + tBlockDataCreate(pBlockData1); + tBlockDataCreate(pBlockData2); for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { code = tsdbReadSubColData(pReader, pBlockIdx, pBlock, iSubBlock, aColId, nCol, pBlockData1, ppBuf1, ppBuf2); if (code) goto _err; code = tBlockDataCopy(pBlockData, pBlockData2); if (code) { - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); goto _err; } code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); if (code) { - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); goto _err; } } - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); } tFree(pBuf1); @@ -1349,34 +1218,34 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl SBlockData *pBlockData1 = &(SBlockData){0}; SBlockData *pBlockData2 = &(SBlockData){0}; - tBlockDataInit(pBlockData1); - tBlockDataInit(pBlockData2); + tBlockDataCreate(pBlockData1); + tBlockDataCreate(pBlockData2); for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { code = tsdbReadSubBlockData(pReader, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); if (code) { - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); goto _err; } code = tBlockDataCopy(pBlockData, pBlockData2); if (code) { - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); goto _err; } // merge two block data code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); if (code) { - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); goto _err; } } - tBlockDataClear(pBlockData1, 1); - tBlockDataClear(pBlockData2, 1); + tBlockDataDestroy(pBlockData1, 1); + tBlockDataDestroy(pBlockData2, 1); } ASSERT(pBlock->nRow == pBlockData->nRow); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 06a38303ec..973cd1e53a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -289,9 +289,9 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type goto _err; } pReader->mBlock = tMapDataInit(); - code = tBlockDataInit(&pReader->oBlockData); + code = tBlockDataCreate(&pReader->oBlockData); if (code) goto _err; - code = tBlockDataInit(&pReader->nBlockData); + code = tBlockDataCreate(&pReader->nBlockData); if (code) goto _err; pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); @@ -327,8 +327,8 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { } taosArrayDestroy(pReader->aBlockIdx); tMapDataClear(&pReader->mBlock); - tBlockDataClear(&pReader->oBlockData, 1); - tBlockDataClear(&pReader->nBlockData, 1); + tBlockDataDestroy(&pReader->oBlockData, 1); + tBlockDataDestroy(&pReader->nBlockData, 1); if (pReader->pDelFReader) { tsdbDelFReaderClose(&pReader->pDelFReader); @@ -1123,7 +1123,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr pWriter->commitID = pTsdb->pVnode->state.commitID; // for data file - code = tBlockDataInit(&pWriter->bData); + code = tBlockDataCreate(&pWriter->bData); if (code) goto _err; pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); @@ -1131,7 +1131,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - code = tBlockDataInit(&pWriter->bDataR); + code = tBlockDataCreate(&pWriter->bDataR); if (code) goto _err; pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx)); @@ -1139,7 +1139,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - code = tBlockDataInit(&pWriter->bDataW); + code = tBlockDataCreate(&pWriter->bDataW); if (code) goto _err; // for del file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 6a27b10cf1..039fe54f3d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1107,7 +1107,7 @@ static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) { } // SBlockData ====================================================== -int32_t tBlockDataInit(SBlockData *pBlockData) { +int32_t tBlockDataCreate(SBlockData *pBlockData) { int32_t code = 0; pBlockData->suid = 0; @@ -1132,7 +1132,7 @@ _exit: return code; } -void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear) { +void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) { tFree((uint8_t *)pBlockData->aUid); tFree((uint8_t *)pBlockData->aVersion); tFree((uint8_t *)pBlockData->aTSKEY); From 8ae2ab1c6e92d044920aa9fed3506acc9e416352 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 05:34:01 +0000 Subject: [PATCH 029/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 31 ++++----- source/dnode/vnode/src/tsdb/tsdbCommit.c | 10 +-- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 13 ++-- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 6 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 66 ++++++++++++++----- 6 files changed, 82 insertions(+), 50 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f33b8cd51d..0cbbe4c351 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -138,22 +138,23 @@ int32_t tGetColData(uint8_t *p, SColData *pColData); #define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) #define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) -int32_t tBlockDataCreate(SBlockData *pBlockData); -void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); - -void tBlockDataReset(SBlockData *pBlockData); -int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid); -int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); -int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); -void tBlockDataClearData(SBlockData *pBlockData); - -int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom); -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); -int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); +int32_t tBlockDataCreate(SBlockData *pBlockData); +void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); +int32_t tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSchema *pTSchema); +int32_t tBlockDataInitEx(SBlockData *pBlockData, int64_t *suid, int64_t uid, SArray *aColId); +void tBlockDataReset(SBlockData *pBlockData); +int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +void tBlockDataClear(SBlockData *pBlockData); SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); -int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); -int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); + +#if 1 +int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); +int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); +int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); +int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); +int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); +#endif // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); @@ -190,7 +191,7 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol uint8_t **ppBuf); int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, uint8_t **ppBuf); -int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size); +int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 44f360d597..d48125beb5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -539,7 +539,7 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { if (code) goto _err; // clear - tBlockDataClearData(pBlockData); + tBlockDataClear(pBlockData); return code; @@ -578,7 +578,7 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { } // clear - tBlockDataClearData(pBlockData); + tBlockDataClear(pBlockData); return code; @@ -596,7 +596,7 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR, NULL, 0); if (code) goto _err; - tBlockDataClearData(pBlockDataW); + tBlockDataClear(pBlockDataW); int32_t iRow = 0; TSDBROW row; TSDBROW *pRow1 = tsdbTbDataIterGet(pIter); @@ -672,7 +672,7 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter STbData *pTbData = pIter->pTbData; SBlockData *pBlockData = &pCommitter->dWriter.bData; - tBlockDataClearData(pBlockData); + tBlockDataClear(pBlockData); TSDBROW *pRow = tsdbTbDataIterGet(pIter); while (true) { if (pRow == NULL) { @@ -741,7 +741,7 @@ static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, S STbData *pTbData = pIter->pTbData; SBlockData *pBlockData = &pCommitter->dWriter.bData; - tBlockDataClearData(pBlockData); + tBlockDataClear(pBlockData); TSDBROW *pRow = tsdbTbDataIterGet(pIter); while (true) { if (pRow == NULL) break; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index cd1e46c342..baafdb488c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1748,7 +1748,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { TSDBKEY key = getCurrentKeyInBuf(pBlockIter, pReader); if (fileBlockShouldLoad(pReader, pFBlock, pBlock, pScanInfo, key)) { tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClearData(&pStatus->fileBlockData); + tBlockDataClear(&pStatus->fileBlockData); code = doLoadFileBlockData(pReader, pBlockIter, pScanInfo, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2208,7 +2208,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn // 3. load the neighbor block, and set it to be the currently accessed file data block tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClearData(&pStatus->fileBlockData); + tBlockDataClear(&pStatus->fileBlockData); int32_t code = doLoadFileBlockData(pReader, pBlockIter, pScanInfo, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2871,7 +2871,7 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClearData(&pStatus->fileBlockData); + tBlockDataClear(&pStatus->fileBlockData); int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, pBlockScanInfo, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData, 1); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 69f39bb3c5..17ab9eff67 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -731,13 +731,14 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo int32_t nColId, SBlockData *pBlockData) { int32_t code = 0; - // TODO - tBlockDataReset(pBlockData); + ASSERT(pBlockData->suid || pBlockData->uid); + + tBlockDataClear(pBlockData); TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; // uid + version + tskey - code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey); + code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey, 1); if (code) goto _err; SDiskDataHdr hdr; uint8_t *p = pReader->pBuf1 + tGetDiskDataHdr(pReader->pBuf1, &hdr); @@ -776,8 +777,8 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo // read and decode columns if (hdr.szBlkCol > 0) { - code = - tsdbReadAndCheckFile(pFD, pBlkInfo->offset + pBlkInfo->szKey, &pReader->pBuf1, hdr.szBlkCol + sizeof(TSCKSUM)); + code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset + pBlkInfo->szKey, &pReader->pBuf1, + hdr.szBlkCol + sizeof(TSCKSUM), 1); if (code) goto _err; int32_t n = 0; @@ -797,7 +798,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo } else { code = tsdbReadAndCheckFile( pFD, pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &pReader->pBuf2, - blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM)); + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM), 1); code = tsdbDecmprColData(pReader->pBuf2, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf3); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 973cd1e53a..4064ad950c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -482,7 +482,7 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { if (code) goto _err; tBlockReset(&pWriter->blockW); - tBlockDataClearData(&pWriter->bDataW); + tBlockDataClear(&pWriter->bDataW); } pWriter->iRow++; @@ -675,7 +675,7 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (code) goto _err; tBlockReset(&pWriter->blockW); - tBlockDataClearData(&pWriter->bDataW); + tBlockDataClear(&pWriter->bDataW); } code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); @@ -725,7 +725,7 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (code) goto _err; tBlockReset(&pWriter->blockW); - tBlockDataClearData(&pWriter->bDataW); + tBlockDataClear(&pWriter->bDataW); } return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 039fe54f3d..92495096da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1145,31 +1145,54 @@ void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) { pBlockData->aColData = NULL; } -void tBlockDataReset(SBlockData *pBlockData) { - pBlockData->suid = 0; - pBlockData->uid = 0; - pBlockData->nRow = 0; - taosArrayClear(pBlockData->aIdx); -} - -int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema, int64_t suid, int64_t uid) { +int32_t tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSchema *pTSchema) { int32_t code = 0; ASSERT(suid || uid); - tBlockDataReset(pBlockData); pBlockData->suid = suid; pBlockData->uid = uid; + pBlockData->nRow = 0; + + taosArrayClear(pBlockData->aIdx); + for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { + STColumn *pTColumn = &pTSchema->columns[iColumn]; + + SColData *pColData; + code = tBlockDataAddColData(pBlockData, iColumn - 1, &pColData); + if (code) goto _exit; + + tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + } + +_exit: + return code; +} + +int32_t tBlockDataInitEx(SBlockData *pBlockData, int64_t *suid, int64_t uid, SArray *aColId) { + int32_t code = 0; + + ASSERT(suid || uid); + + pBlockData->suid = suid; + pBlockData->uid = uid; + pBlockData->nRow = 0; + + taosArrayClear(pBlockData->aIdx); + if (aColId) { + int16_t lcid = -1; + for (int32_t iColId = 0; iColId < taosArrayGetSize(aColId); iColId++) { + int16_t cid = *(int16_t *)taosArrayGet(aColId, iColId); + + ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(cid > lcid); + lcid = cid; - if (pTSchema) { - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - STColumn *pTColumn = &pTSchema->columns[iColumn]; SColData *pColData; - - code = tBlockDataAddColData(pBlockData, iColumn - 1, &pColData); + code = tBlockDataAddColData(pBlockData, iColId, &pColData); if (code) goto _exit; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) != 0); + tColDataInit(pColData, cid, TSDB_DATA_TYPE_NULL, -1); } } @@ -1177,7 +1200,14 @@ _exit: return code; } -void tBlockDataClearData(SBlockData *pBlockData) { +void tBlockDataReset(SBlockData *pBlockData) { + pBlockData->suid = 0; + pBlockData->uid = 0; + pBlockData->nRow = 0; + taosArrayClear(pBlockData->aIdx); +} + +void tBlockDataClear(SBlockData *pBlockData) { pBlockData->nRow = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1869,7 +1899,7 @@ _exit: return code; } -int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size) { +int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) { int32_t code = 0; // alloc @@ -1894,7 +1924,7 @@ int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int } // check - if (!taosCheckChecksumWhole(*ppOut, size)) { + if (toCheck && !taosCheckChecksumWhole(*ppOut, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _exit; } From 6baae7d50fd70707382d24de0f3fea78d1425978 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 06:43:51 +0000 Subject: [PATCH 030/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 2 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 78 ++++++++++++------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 9 ++- 3 files changed, 57 insertions(+), 32 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0cbbe4c351..2606d7a8d0 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -191,7 +191,7 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol uint8_t **ppBuf); int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, uint8_t **ppBuf); -int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck); +int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 17ab9eff67..48023858cc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -727,27 +727,24 @@ _err: return code; } -static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, int8_t fromLast, int16_t *aColId, - int32_t nColId, SBlockData *pBlockData) { +static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, int8_t fromLast, + SBlockData *pBlockData) { int32_t code = 0; - ASSERT(pBlockData->suid || pBlockData->uid); - tBlockDataClear(pBlockData); TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; // uid + version + tskey - code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey, 1); + code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey, 1); if (code) goto _err; SDiskDataHdr hdr; uint8_t *p = pReader->pBuf1 + tGetDiskDataHdr(pReader->pBuf1, &hdr); ASSERT(hdr.delimiter == TSDB_FILE_DLMT); - ASSERT(hdr.suid || hdr.uid); + ASSERT(pBlockData->suid == hdr.suid); + ASSERT(pBlockData->uid == hdr.uid); - pBlockData->suid = hdr.suid; - pBlockData->uid = hdr.uid; pBlockData->nRow = hdr.nRow; // uid @@ -776,36 +773,61 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo ASSERT(p - pReader->pBuf1 == sizeof(TSCKSUM)); // read and decode columns + if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit; + if (hdr.szBlkCol > 0) { - code = tsdbReadAndCheckFile(pFD, pBlkInfo->offset + pBlkInfo->szKey, &pReader->pBuf1, - hdr.szBlkCol + sizeof(TSCKSUM), 1); + int64_t offset = pBlkInfo->offset + pBlkInfo->szKey; + code = tsdbReadAndCheck(pFD, offset, &pReader->pBuf1, hdr.szBlkCol + sizeof(TSCKSUM), 1); if (code) goto _err; + } - int32_t n = 0; - while (n < hdr.szBlkCol) { - SBlockCol blockCol; + SBlockCol blockCol = {.cid = 0}; + SBlockCol *pBlockCol = &blockCol; + int32_t n = 0; - n += tGetBlockCol(pReader->pBuf1 + n, &blockCol); + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - ASSERT(blockCol.flag && blockCol.flag != HAS_NONE); - - // TODO: merge with the column IDs - - SColData *pColData = NULL; // (todo) - - if (blockCol.flag == HAS_NULL) { - // TODO: make a hdr.nRow COL_VAL_NULL(); + while (pBlockCol && pBlockCol->cid < pColData->cid) { + if (n < hdr.szBlkCol) { + n += tGetBlockCol(pReader->pBuf1 + n, pBlockCol); } else { - code = tsdbReadAndCheckFile( - pFD, pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &pReader->pBuf2, - blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM), 1); + ASSERT(n == hdr.szBlkCol); + pBlockCol = NULL; + } + } - code = tsdbDecmprColData(pReader->pBuf2, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf3); + if (pBlockCol == NULL || pBlockCol->cid > pColData->cid) { + // add a lot of NONE + for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pBlockCol->cid, pBlockCol->type)); + if (code) goto _err; + } + } else { + ASSERT(pBlockCol->type == pColData->type); + ASSERT(pBlockCol->flag && pBlockCol->flag != HAS_NONE); + + if (pBlockCol->flag == HAS_NULL) { + // add a lot of NULL + for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NULL(pBlockCol->cid, pBlockCol->type)); + if (code) goto _err; + } + } else { + // decode from binary + int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + pBlockCol->offset; + int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); + + code = tsdbReadAndCheck(pFD, offset, &pReader->pBuf2, size, 0); + if (code) goto _err; + + code = tsdbDecmprColData(pReader->pBuf2, pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf3); if (code) goto _err; } } } +_exit: return code; _err: @@ -817,7 +839,7 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl int32_t nColId) { int32_t code = 0; - code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, aColId, nColId, pBlockData); + code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, pBlockData); if (code) goto _err; for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { @@ -836,7 +858,7 @@ int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *p int32_t nColId) { int32_t code = 0; - code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, aColId, nColId, pBlockData); + code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, pBlockData); if (code) goto _err; return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 92495096da..e4ebc3e839 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1172,6 +1172,7 @@ _exit: int32_t tBlockDataInitEx(SBlockData *pBlockData, int64_t *suid, int64_t uid, SArray *aColId) { int32_t code = 0; + ASSERT(0); ASSERT(suid || uid); pBlockData->suid = suid; @@ -1208,6 +1209,8 @@ void tBlockDataReset(SBlockData *pBlockData) { } void tBlockDataClear(SBlockData *pBlockData) { + ASSERT(pBlockData->suid || pBlockData->uid); + pBlockData->nRow = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1864,8 +1867,8 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in goto _exit; } - pColData->cid = pBlockCol->cid; - pColData->type = pBlockCol->type; + ASSERT(pColData->cid == pBlockCol->cid); + ASSERT(pColData->type == pBlockCol->type); pColData->smaOn = pBlockCol->smaOn; pColData->flag = pBlockCol->flag; pColData->nVal = nVal; @@ -1899,7 +1902,7 @@ _exit: return code; } -int32_t tsdbReadAndCheckFile(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) { +int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) { int32_t code = 0; // alloc From 49fde0b1a87937a3c6c94551c51f2ad94d6c95af Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 07:27:44 +0000 Subject: [PATCH 031/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 12 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 593 ++---------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 206 +++--- 3 files changed, 139 insertions(+), 672 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2606d7a8d0..0e10d1b000 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -141,17 +141,17 @@ int32_t tGetColData(uint8_t *p, SColData *pColData); int32_t tBlockDataCreate(SBlockData *pBlockData); void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); int32_t tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSchema *pTSchema); -int32_t tBlockDataInitEx(SBlockData *pBlockData, int64_t *suid, int64_t uid, SArray *aColId); +int32_t tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom); void tBlockDataReset(SBlockData *pBlockData); int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); void tBlockDataClear(SBlockData *pBlockData); SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); +int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); +int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); #if 1 int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); -int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); #endif @@ -261,10 +261,8 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, int16_t *aColId, - int32_t nColId); -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, int16_t *aColId, - int32_t nColId); +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData); +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData); // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 48023858cc..f73883d050 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -835,16 +835,51 @@ _err: return code; } -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, int16_t *aColId, - int32_t nColId) { +int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData) { int32_t code = 0; code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, pBlockData); if (code) goto _err; - for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - // TODO - ASSERT(0); + if (pBlock->nSubBlock > 1) { + SBlockData bData1; + SBlockData bData2; + + // create + code = tBlockDataCreate(&bData1); + if (code) goto _err; + code = tBlockDataCreate(&bData2); + if (code) goto _err; + + // init + tBlockDataInitEx(&bData1, pBlockData); + tBlockDataInitEx(&bData2, pBlockData); + + for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { + code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[iSubBlock], 0, &bData1); + if (code) { + tBlockDataDestroy(&bData1, 1); + tBlockDataDestroy(&bData2, 1); + goto _err; + } + + code = tBlockDataCopy(pBlockData, &bData2); + if (code) { + tBlockDataDestroy(&bData1, 1); + tBlockDataDestroy(&bData2, 1); + goto _err; + } + + code = tBlockDataMerge(&bData1, &bData2, pBlockData); + if (code) { + tBlockDataDestroy(&bData1, 1); + tBlockDataDestroy(&bData2, 1); + goto _err; + } + } + + tBlockDataDestroy(&bData1, 1); + tBlockDataDestroy(&bData2, 1); } return code; @@ -854,8 +889,7 @@ _err: return code; } -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, int16_t *aColId, - int32_t nColId) { +int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData) { int32_t code = 0; code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, pBlockData); @@ -868,551 +902,6 @@ _err: return code; } -#if 0 -static int32_t tsdbReadColDataImpl(SBlockInfo *pSubBlock, SBlockCol *pBlockCol, SColData *pColData, uint8_t *pBuf, - uint8_t **ppBuf) { - int32_t code = 0; -#if 0 - int64_t size; - int64_t n; - - if (!taosCheckChecksumWhole(pBuf, pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - pColData->nVal = pSubBlock->nRow; - pColData->flag = pBlockCol->flag; - - // BITMAP - if (pBlockCol->flag != HAS_VALUE) { - ASSERT(pBlockCol->szBitmap); - - size = BIT2_SIZE(pColData->nVal); - code = tRealloc(&pColData->pBitMap, size); - if (code) goto _err; - - code = tRealloc(ppBuf, size + COMP_OVERFLOW_BYTES); - if (code) goto _err; - - n = tsDecompressTinyint(pBuf, pBlockCol->szBitmap, size, pColData->pBitMap, size, TWO_STAGE_COMP, *ppBuf, - size + COMP_OVERFLOW_BYTES); - if (n <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - ASSERT(n == size); - } else { - ASSERT(pBlockCol->szBitmap == 0); - } - pBuf = pBuf + pBlockCol->szBitmap; - - // OFFSET - if (IS_VAR_DATA_TYPE(pColData->type)) { - ASSERT(pBlockCol->szOffset); - - size = sizeof(int32_t) * pColData->nVal; - code = tRealloc((uint8_t **)&pColData->aOffset, size); - if (code) goto _err; - - code = tRealloc(ppBuf, size + COMP_OVERFLOW_BYTES); - if (code) goto _err; - - n = tsDecompressInt(pBuf, pBlockCol->szOffset, pColData->nVal, (char *)pColData->aOffset, size, TWO_STAGE_COMP, - *ppBuf, size + COMP_OVERFLOW_BYTES); - if (n <= 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - ASSERT(n == size); - } else { - ASSERT(pBlockCol->szOffset == 0); - } - pBuf = pBuf + pBlockCol->szOffset; - - // VALUE - pColData->nData = pBlockCol->szOrigin; - - code = tRealloc(&pColData->pData, pColData->nData); - if (code) goto _err; - - if (pSubBlock->cmprAlg == NO_COMPRESSION) { - memcpy(pColData->pData, pBuf, pColData->nData); - } else { - if (pSubBlock->cmprAlg == TWO_STAGE_COMP) { - code = tRealloc(ppBuf, pColData->nData + COMP_OVERFLOW_BYTES); - if (code) goto _err; - } - - n = tDataTypes[pBlockCol->type].decompFunc(pBuf, pBlockCol->szValue, pSubBlock->nRow, pColData->pData, - pColData->nData, pSubBlock->cmprAlg, *ppBuf, - pColData->nData + COMP_OVERFLOW_BYTES); - if (n < 0) { - code = TSDB_CODE_COMPRESS_ERROR; - goto _err; - } - - ASSERT(n == pColData->nData); - } - - return code; - -_err: -#endif - return code; -} - -static int32_t tsdbReadSubColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int32_t iSubBlock, - int16_t *aColId, int32_t nCol, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2) { - int32_t code = 0; -#if 0 - TdFilePtr pFD = pReader->pDataFD; - SBlockInfo *pSubBlock = &pBlock->aSubBlock[iSubBlock]; - SArray *aBlockCol = NULL; - int64_t offset; - int64_t size; - int64_t n; - - tBlockDataReset(pBlockData); - pBlockData->nRow = pSubBlock->nRow; - - // TSDBKEY and SBlockCol - if (nCol == 1) { - offset = pSubBlock->offset + pSubBlock->szBlockCol + sizeof(TSCKSUM); - size = pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); - } else { - offset = pSubBlock->offset; - size = pSubBlock->szBlockCol + sizeof(TSCKSUM) + pSubBlock->szVersion + pSubBlock->szTSKEY + sizeof(TSCKSUM); - } - - code = tRealloc(ppBuf1, size); - if (code) goto _err; - - n = taosLSeekFile(pFD, offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosReadFile(pFD, *ppBuf1, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - if (nCol == 1) { - code = tsdbReadBlockDataKey(pBlockData, pSubBlock, *ppBuf1, ppBuf2); - if (code) goto _err; - - goto _exit; - } else { - aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - code = tsdbReadBlockCol(*ppBuf1, pSubBlock->szBlock, NULL /*todo*/, aBlockCol); - if (code) goto _err; - - code = tsdbReadBlockDataKey(pBlockData, pSubBlock, *ppBuf1 + pSubBlock->szBlockCol + sizeof(TSCKSUM), ppBuf2); - if (code) goto _err; - } - - for (int32_t iCol = 1; iCol < nCol; iCol++) { - void *p = taosArraySearch(aBlockCol, &(SBlockCol){.cid = aColId[iCol]}, tBlockColCmprFn, TD_EQ); - - if (p) { - SBlockCol *pBlockCol = (SBlockCol *)p; - SColData *pColData; - - ASSERT(pBlockCol->flag && pBlockCol->flag != HAS_NONE); - - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _err; - - tColDataInit(pColData, pBlockCol->cid, pBlockCol->type, pBlockCol->smaOn); - if (pBlockCol->flag == HAS_NULL) { - for (int32_t iRow = 0; iRow < pSubBlock->nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pBlockCol->cid, pBlockCol->type)); - if (code) goto _err; - } - } else { - offset = pSubBlock->offset + pSubBlock->szBlockCol + sizeof(TSCKSUM) + pSubBlock->szVersion + - pSubBlock->szTSKEY + sizeof(TSCKSUM) + pBlockCol->offset; - size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); - - code = tRealloc(ppBuf1, size); - if (code) goto _err; - - // seek - n = taosLSeekFile(pFD, offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // read - n = taosReadFile(pFD, *ppBuf1, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - code = tsdbReadColDataImpl(pSubBlock, pBlockCol, pColData, *ppBuf1, ppBuf2); - if (code) goto _err; - } - } - } - -_exit: - taosArrayDestroy(aBlockCol); - return code; - -_err: - taosArrayDestroy(aBlockCol); -#endif - return code; -} - -int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol, - SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2) { - int32_t code = 0; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - - ASSERT(aColId[0] == PRIMARYKEY_TIMESTAMP_COL_ID); - - if (!ppBuf1) ppBuf1 = &pBuf1; - if (!ppBuf2) ppBuf2 = &pBuf2; - - code = tsdbReadSubColData(pReader, pBlockIdx, pBlock, 0, aColId, nCol, pBlockData, ppBuf1, ppBuf2); - if (code) goto _err; - - if (pBlock->nSubBlock > 1) { - SBlockData *pBlockData1 = &(SBlockData){0}; - SBlockData *pBlockData2 = &(SBlockData){0}; - - tBlockDataCreate(pBlockData1); - tBlockDataCreate(pBlockData2); - for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - code = tsdbReadSubColData(pReader, pBlockIdx, pBlock, iSubBlock, aColId, nCol, pBlockData1, ppBuf1, ppBuf2); - if (code) goto _err; - - code = tBlockDataCopy(pBlockData, pBlockData2); - if (code) { - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - goto _err; - } - - code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); - if (code) { - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - goto _err; - } - } - - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - } - - tFree(pBuf1); - tFree(pBuf2); - return code; - -_err: - tsdbError("vgId:%d, tsdb read col data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf1); - tFree(pBuf2); - return code; -} - -static int32_t tsdbReadSubBlockData(SDataFReader *pReader, SBlock *pBlock, int32_t iSubBlock, SBlockData *pBlockData, - uint8_t **ppBuf1, uint8_t **ppBuf2) { - int32_t code = 0; -#if 0 - uint8_t *p; - int64_t size; - int64_t n; - TdFilePtr pFD = pReader->pDataFD; - SBlockInfo *pSubBlock = &pBlock->aSubBlock[iSubBlock]; - SArray *aBlockCol = NULL; - - tBlockDataReset(pBlockData); - - // realloc - code = tRealloc(ppBuf1, pSubBlock->szBlock); - if (code) goto _err; - - // seek - n = taosLSeekFile(pFD, pSubBlock->offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // read - n = taosReadFile(pFD, *ppBuf1, pSubBlock->szBlock); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < pSubBlock->szBlock) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - pBlockData->nRow = pSubBlock->nRow; - - // TSDBKEY - p = *ppBuf1 + pSubBlock->szBlockCol + sizeof(TSCKSUM); - code = tsdbReadBlockDataKey(pBlockData, pSubBlock, p, ppBuf2); - if (code) goto _err; - - // COLUMNS - aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - code = tsdbReadBlockCol(pSubBlock, *ppBuf1, aBlockCol); - if (code) goto _err; - - for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(aBlockCol); iBlockCol++) { - SColData *pColData; - SBlockCol *pBlockCol = (SBlockCol *)taosArrayGet(aBlockCol, iBlockCol); - - ASSERT(pBlockCol->flag && pBlockCol->flag != HAS_NONE); - - code = tBlockDataAddColData(pBlockData, iBlockCol, &pColData); - if (code) goto _err; - - tColDataInit(pColData, pBlockCol->cid, pBlockCol->type, pBlockCol->smaOn); - if (pBlockCol->flag == HAS_NULL) { - for (int32_t iRow = 0; iRow < pSubBlock->nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pBlockCol->cid, pBlockCol->type)); - if (code) goto _err; - } - } else { - p = *ppBuf1 + pSubBlock->szBlockCol + sizeof(TSCKSUM) + pSubBlock->szVersion + pSubBlock->szTSKEY + - sizeof(TSCKSUM) + pBlockCol->offset; - code = tsdbReadColDataImpl(pSubBlock, pBlockCol, pColData, p, ppBuf2); - if (code) goto _err; - } - } - - taosArrayDestroy(aBlockCol); - return code; - -_err: - tsdbError("vgId:%d, tsdb read sub block data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - taosArrayDestroy(aBlockCol); -#endif - return code; -} - -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2) { - int32_t code = 0; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - - if (!ppBuf1) ppBuf1 = &pBuf1; - if (!ppBuf2) ppBuf2 = &pBuf2; - - // read the first sub-block - int32_t iSubBlock = 0; - code = tsdbReadSubBlockData(pReader, pBlock, iSubBlock, pBlockData, ppBuf1, ppBuf2); - if (code) goto _err; - - // read remain block data and do merg - if (pBlock->nSubBlock > 1) { - SBlockData *pBlockData1 = &(SBlockData){0}; - SBlockData *pBlockData2 = &(SBlockData){0}; - - tBlockDataCreate(pBlockData1); - tBlockDataCreate(pBlockData2); - for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - code = tsdbReadSubBlockData(pReader, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); - if (code) { - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - goto _err; - } - - code = tBlockDataCopy(pBlockData, pBlockData2); - if (code) { - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - goto _err; - } - - // merge two block data - code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); - if (code) { - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - goto _err; - } - } - - tBlockDataDestroy(pBlockData1, 1); - tBlockDataDestroy(pBlockData2, 1); - } - - ASSERT(pBlock->nRow == pBlockData->nRow); - ASSERT(tsdbKeyCmprFn(&pBlock->minKey, &TSDBROW_KEY(&tBlockDataFirstRow(pBlockData))) == 0); - ASSERT(tsdbKeyCmprFn(&pBlock->maxKey, &TSDBROW_KEY(&tBlockDataLastRow(pBlockData))) == 0); - - tFree(pBuf1); - tFree(pBuf2); - return code; - -_err: - tsdbError("vgId:%d, tsdb read block data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); - tFree(pBuf1); - tFree(pBuf2); - return code; -} - -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData, uint8_t **ppBuf1, - uint8_t **ppBuf2) { - int32_t code = 0; -#if 0 - - tBlockDataReset(pBlockData); - - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - if (!ppBuf1) ppBuf1 = &pBuf1; - if (!ppBuf2) ppBuf2 = &pBuf2; - - // realloc - code = tRealloc(ppBuf1, pBlockL->szBlock); - if (code) goto _err; - - // seek - int64_t n = taosLSeekFile(pReader->pLastFD, pBlockL->offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // read - n = taosReadFile(pReader->pLastFD, *ppBuf1, pBlockL->szBlock); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < pBlockL->szBlock) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // decode block col - SDiskDataHdr hdr; - SArray *aBlockCol = taosArrayInit(0, sizeof(SBlockCol)); - uint8_t *p = *ppBuf1; - if (aBlockCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - code = tsdbReadBlockCol(p, pBlockL->szBlockCol, &hdr, aBlockCol); - if (code) goto _err; - p += pBlockL->szBlockCol + sizeof(TSCKSUM); - - // checksum - if (!taosCheckChecksumWhole(p, pBlockL->szUid + pBlockL->szVer + pBlockL->szTSKEY + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // UID - if (hdr.uid == 0) { - code = tsdbReadDataArray(p, pBlockL->szUid, pBlockL->nRow, TSDB_DATA_TYPE_BIGINT, pBlockL->cmprAlg, - (uint8_t **)&pBlockData->aUid, ppBuf2); - if (code) goto _err; - } else { - ASSERT(pBlockL->szUid == 0); - } - p += pBlockL->szUid; - - // VERSION - code = tsdbReadDataArray(p, pBlockL->szVer, pBlockL->nRow, TSDB_DATA_TYPE_BIGINT, pBlockL->cmprAlg, - (uint8_t **)&pBlockData->aVersion, ppBuf2); - if (code) goto _err; - p += pBlockL->szVer; - - // TSKEY - code = tsdbReadDataArray(p, pBlockL->szTSKEY, pBlockL->nRow, TSDB_DATA_TYPE_TIMESTAMP, pBlockL->cmprAlg, - (uint8_t **)&pBlockData->aTSKEY, ppBuf2); - if (code) goto _err; - p += pBlockL->szTSKEY; - p += sizeof(TSCKSUM); - - // COLUMN - code = tBlockDataSetSchema(pBlockData, NULL, hdr.suid, hdr.uid); - if (code) goto _err; - - for (int32_t iBlockCol = 0; iBlockCol < taosArrayGetSize(aBlockCol); iBlockCol++) { - SBlockCol *pBlockCol = (SBlockCol *)taosArrayGet(aBlockCol, iBlockCol); - SColData *pColData; - - // checksum - if (!taosCheckChecksumWhole(p, pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // add SColData - code = tBlockDataAddColData(pBlockData, iBlockCol, &pColData); - if (code) goto _err; - tColDataInit(pColData, pBlockCol->cid, pBlockCol->type, pBlockCol->smaOn); - pColData->nVal = pBlockL->nRow; - pColData->flag = pBlockCol->flag; - - // bitmap - if (pBlockCol->szBitmap) { - code = tsdbReadDataArray(p, pBlockCol->szBitmap, ); - if (code) goto _err; - } - p += pBlockCol->szBitmap; - - // offset - if (pBlockCol->szOffset) { - code = tsdbReadDataArray(p, pBlockCol->szOffset, ); - if (code) goto _err; - } - p += pBlockCol->szOffset; - - // value - pColData->nData = pBlockCol->szOrigin; - if (pColData->nData) { - // TODO - } - } - - taosArrayDestroy(aBlockCol); - - return code; - -_err: - tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); -#endif - return code; -} -#endif - // SDataFWriter ==================================================== int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index e4ebc3e839..d0d354e467 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -949,13 +949,14 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { int32_t size; ASSERT(pColDataSrc->nVal > 0); + ASSERT(pColDataDest->cid = pColDataSrc->cid); + ASSERT(pColDataDest->type = pColDataSrc->type); - pColDataDest->cid = pColDataSrc->cid; - pColDataDest->type = pColDataSrc->type; pColDataDest->smaOn = pColDataSrc->smaOn; pColDataDest->nVal = pColDataSrc->nVal; pColDataDest->flag = pColDataSrc->flag; + // bitmap if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { size = BIT2_SIZE(pColDataSrc->nVal); code = tRealloc(&pColDataDest->pBitMap, size); @@ -963,6 +964,7 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); } + // offset if (IS_VAR_DATA_TYPE(pColDataDest->type)) { size = sizeof(int32_t) * pColDataSrc->nVal; @@ -972,9 +974,10 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); } + // value + pColDataDest->nData = pColDataSrc->nData; code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); if (code) goto _exit; - pColDataDest->nData = pColDataSrc->nData; memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); _exit: @@ -1169,32 +1172,25 @@ _exit: return code; } -int32_t tBlockDataInitEx(SBlockData *pBlockData, int64_t *suid, int64_t uid, SArray *aColId) { +int32_t tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom) { int32_t code = 0; ASSERT(0); - ASSERT(suid || uid); + ASSERT(pBlockDataFrom->suid || pBlockDataFrom->uid); - pBlockData->suid = suid; - pBlockData->uid = uid; + pBlockData->suid = pBlockDataFrom->suid; + pBlockData->uid = pBlockDataFrom->uid; pBlockData->nRow = 0; taosArrayClear(pBlockData->aIdx); - if (aColId) { - int16_t lcid = -1; - for (int32_t iColId = 0; iColId < taosArrayGetSize(aColId); iColId++) { - int16_t cid = *(int16_t *)taosArrayGet(aColId, iColId); + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockDataFrom->aIdx); iColData++) { + SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColData); - ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID); - ASSERT(cid > lcid); - lcid = cid; + SColData *pColData; + code = tBlockDataAddColData(pBlockData, iColData, &pColData); + if (code) goto _exit; - SColData *pColData; - code = tBlockDataAddColData(pBlockData, iColId, &pColData); - if (code) goto _exit; - - tColDataInit(pColData, cid, TSDB_DATA_TYPE_NULL, -1); - } + tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn); } _exit: @@ -1339,128 +1335,112 @@ _exit: int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData) { int32_t code = 0; - // set target - int32_t iColData1 = 0; - int32_t nColData1 = taosArrayGetSize(pBlockData1->aIdx); - int32_t iColData2 = 0; - int32_t nColData2 = taosArrayGetSize(pBlockData2->aIdx); - SColData *pColData1; - SColData *pColData2; - SColData *pColData; + ASSERT(pBlockData->suid == pBlockData1->suid); + ASSERT(pBlockData->uid == pBlockData1->uid); + ASSERT(pBlockData1->nRow > 0); + ASSERT(pBlockData2->nRow > 0); - tBlockDataReset(pBlockData); - while (iColData1 < nColData1 && iColData2 < nColData2) { - pColData1 = tBlockDataGetColDataByIdx(pBlockData1, iColData1); - pColData2 = tBlockDataGetColDataByIdx(pBlockData2, iColData2); + tBlockDataClear(pBlockData); - if (pColData1->cid == pColData2->cid) { - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _exit; - tColDataInit(pColData, pColData2->cid, pColData2->type, pColData2->smaOn); + TSDBROW row1 = tsdbRowFromBlockData(pBlockData1, 0); + TSDBROW row2 = tsdbRowFromBlockData(pBlockData2, 0); + TSDBROW *pRow1 = &row1; + TSDBROW *pRow2 = &row2; - iColData1++; - iColData2++; - } else if (pColData1->cid < pColData2->cid) { - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _exit; - tColDataInit(pColData, pColData1->cid, pColData1->type, pColData1->smaOn); + while (pRow1 && pRow2) { + int32_t c = tsdbRowCmprFn(pRow1, pRow2); - iColData1++; - } else { - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _exit; - tColDataInit(pColData, pColData2->cid, pColData2->type, pColData2->smaOn); - - iColData2++; - } - } - - while (iColData1 < nColData1) { - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _exit; - tColDataInit(pColData, pColData1->cid, pColData1->type, pColData1->smaOn); - - iColData1++; - } - - while (iColData2 < nColData2) { - code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); - if (code) goto _exit; - tColDataInit(pColData, pColData2->cid, pColData2->type, pColData2->smaOn); - - iColData2++; - } - - // loop to merge - int32_t iRow1 = 0; - int32_t nRow1 = pBlockData1->nRow; - int32_t iRow2 = 0; - int32_t nRow2 = pBlockData2->nRow; - TSDBROW row1; - TSDBROW row2; - int32_t c; - - while (iRow1 < nRow1 && iRow2 < nRow2) { - row1 = tsdbRowFromBlockData(pBlockData1, iRow1); - row2 = tsdbRowFromBlockData(pBlockData2, iRow2); - - c = tsdbKeyCmprFn(&TSDBROW_KEY(&row1), &TSDBROW_KEY(&row2)); if (c < 0) { - // code = tBlockDataAppendRow(pBlockData, &row1, NULL); + code = tBlockDataAppendRow(pBlockData, pRow1, NULL, + pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); if (code) goto _exit; - iRow1++; + + pRow1->iRow++; + if (pRow1->iRow < pBlockData1->nRow) { + *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); + } else { + pRow1 = NULL; + } } else if (c > 0) { - // code = tBlockDataAppendRow(pBlockData, &row2, NULL); + code = tBlockDataAppendRow(pBlockData, pRow2, NULL, + pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); if (code) goto _exit; - iRow2++; + + pRow2->iRow++; + if (pRow2->iRow < pBlockData2->nRow) { + *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); + } else { + pRow2 = NULL; + } } else { ASSERT(0); } } - while (iRow1 < nRow1) { - row1 = tsdbRowFromBlockData(pBlockData1, iRow1); - // code = tBlockDataAppendRow(pBlockData, &row1, NULL); + while (pRow1) { + code = tBlockDataAppendRow(pBlockData, pRow1, NULL, + pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); if (code) goto _exit; - iRow1++; + + pRow1->iRow++; + if (pRow1->iRow < pBlockData1->nRow) { + *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); + } else { + pRow1 = NULL; + } } - while (iRow2 < nRow2) { - row2 = tsdbRowFromBlockData(pBlockData2, iRow2); - // code = tBlockDataAppendRow(pBlockData, &row2, NULL); + while (pRow2) { + code = tBlockDataAppendRow(pBlockData, pRow2, NULL, + pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); if (code) goto _exit; - iRow2++; + + pRow2->iRow++; + if (pRow2->iRow < pBlockData2->nRow) { + *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); + } else { + pRow2 = NULL; + } } _exit: return code; } -int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest) { - int32_t code = 0; - SColData *pColDataSrc; - SColData *pColDataDest; +int32_t tBlockDataCopy(SBlockData *pSrc, SBlockData *pDest) { + int32_t code = 0; - ASSERT(pBlockDataSrc->nRow > 0); + tBlockDataClear(pDest); - tBlockDataReset(pBlockDataDest); + ASSERT(pDest->suid == pSrc->suid); + ASSERT(pDest->uid == pSrc->uid); + ASSERT(pSrc->nRow == pDest->nRow); + ASSERT(taosArrayGetSize(pSrc->aIdx) == taosArrayGetSize(pDest->aIdx)); - pBlockDataDest->nRow = pBlockDataSrc->nRow; - // TSDBKEY - code = tRealloc((uint8_t **)&pBlockDataDest->aVersion, sizeof(int64_t) * pBlockDataSrc->nRow); - if (code) goto _exit; - code = tRealloc((uint8_t **)&pBlockDataDest->aTSKEY, sizeof(TSKEY) * pBlockDataSrc->nRow); - if (code) goto _exit; - memcpy(pBlockDataDest->aVersion, pBlockDataSrc->aVersion, sizeof(int64_t) * pBlockDataSrc->nRow); - memcpy(pBlockDataDest->aTSKEY, pBlockDataSrc->aTSKEY, sizeof(TSKEY) * pBlockDataSrc->nRow); + pDest->nRow = pSrc->nRow; - // other - for (size_t iColData = 0; iColData < taosArrayGetSize(pBlockDataSrc->aIdx); iColData++) { - pColDataSrc = tBlockDataGetColDataByIdx(pBlockDataSrc, iColData); - code = tBlockDataAddColData(pBlockDataDest, iColData, &pColDataDest); + if (pSrc->uid == 0) { + code = tRealloc((uint8_t **)&pDest->aUid, sizeof(int64_t) * pDest->nRow); if (code) goto _exit; + memcpy(pDest->aUid, pSrc->aUid, sizeof(int64_t) * pDest->nRow); + } - code = tColDataCopy(pColDataSrc, pColDataDest); + code = tRealloc((uint8_t **)&pDest->aVersion, sizeof(int64_t) * pDest->nRow); + if (code) goto _exit; + memcpy(pDest->aVersion, pSrc->aVersion, sizeof(int64_t) * pDest->nRow); + + code = tRealloc((uint8_t **)&pDest->aTSKEY, sizeof(TSKEY) * pDest->nRow); + if (code) goto _exit; + memcpy(pDest->aTSKEY, pSrc->aTSKEY, sizeof(TSKEY) * pDest->nRow); + + for (int32_t iColData = 0; iColData < taosArrayGetSize(pSrc->aIdx); iColData++) { + SColData *pColSrc = tBlockDataGetColDataByIdx(pSrc, iColData); + SColData *pColDest = tBlockDataGetColDataByIdx(pDest, iColData); + + ASSERT(pColSrc->cid == pColDest->cid); + ASSERT(pColSrc->type == pColDest->type); + + code = tColDataCopy(pColSrc, pColDest); if (code) goto _exit; } From 4f06c4bbddbc64c3d6facc4921aa4576e09d1a3f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 07:32:02 +0000 Subject: [PATCH 032/194] make it compile --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 12 ++++++------ source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 2cb3f1f46d..3cff6bef27 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -507,7 +507,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetBlock); /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ - code = tsdbReadDataBlock(state->pDataFReader, &block, state->pBlockData, NULL, 0); + code = tsdbReadDataBlock(state->pDataFReader, &block, state->pBlockData); if (code) goto _err; state->nRow = state->blockData.nRow; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d48125beb5..108a138d34 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -309,7 +309,7 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { pCommitter->dReader.iBlockL++; if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal, NULL, 0); + code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal); if (code) goto _exit; pCommitter->dReader.iRow = 0; @@ -593,7 +593,7 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, S SBlockData *pBlockDataR = &pCommitter->dReader.bData; SBlockData *pBlockDataW = &pCommitter->dWriter.bData; - code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR, NULL, 0); + code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR); if (code) goto _err; tBlockDataClear(pBlockDataW); @@ -823,8 +823,8 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { // set block data schema if need if (pBlockData->suid == 0 && pBlockData->uid == 0) { - code = tBlockDataSetSchema(pBlockData, pCommitter->skmTable.pTSchema, pTbData->suid, - pTbData->suid ? 0 : pTbData->uid); + code = + tBlockDataInit(pBlockData, pTbData->suid, pTbData->suid ? 0 : pTbData->uid, pCommitter->skmTable.pTSchema); if (code) goto _err; } @@ -966,7 +966,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { if (code) goto _err; tMapDataReset(&pCommitter->dWriter.mBlock); - code = tBlockDataSetSchema(&pCommitter->dWriter.bData, pCommitter->skmTable.pTSchema, pTbData->suid, pTbData->uid); + code = tBlockDataInit(&pCommitter->dWriter.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); if (code) goto _err; // .data merge @@ -1144,7 +1144,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid, 1 /*TOOD*/); if (code) goto _err; - code = tBlockDataSetSchema(pBlockDataW, pCommitter->skmTable.pTSchema, suid, suid ? 0 : uid); + code = tBlockDataInit(pBlockDataW, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); if (code) goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index baafdb488c..73454e2a63 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -785,7 +785,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockIdx blockIdx = {.suid = pReader->suid, .uid = pBlockScanInfo->uid}; - int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData, pSupInfo->colIds, numOfCols); + int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData /*, pSupInfo->colIds, numOfCols*/); if (code != TSDB_CODE_SUCCESS) { goto _error; } From f37a072e77e44ab0bba19713c4dad493b4350860 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 10:06:07 +0000 Subject: [PATCH 033/194] more work --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 147 +++++++++++++---------- 1 file changed, 81 insertions(+), 66 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 108a138d34..2395a4cd8b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -53,7 +53,6 @@ typedef struct { // last SArray *aBlockL; // SArray int32_t iBlockL; - SBlockL *pBlockL; SBlockData bDatal; int32_t iRow; SRowInfo *pRowInfo; @@ -290,6 +289,54 @@ _err: return code; } +static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { + int32_t code = 0; + + if (pCommitter->skmTable.pTSchema) { + if (pCommitter->skmTable.suid == suid) { + if (suid == 0) { + if (pCommitter->skmTable.uid == uid && sver == pCommitter->skmTable.pTSchema->version) goto _exit; + } else { + if (sver == pCommitter->skmTable.pTSchema->version) goto _exit; + } + } + } + + pCommitter->skmTable.suid = suid; + pCommitter->skmTable.uid = uid; + tTSchemaDestroy(pCommitter->skmTable.pTSchema); + code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmTable.pTSchema); + if (code) goto _exit; + +_exit: + return code; +} + +static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { + int32_t code = 0; + + if (pCommitter->skmRow.pTSchema) { + if (pCommitter->skmRow.suid == suid) { + if (suid == 0) { + if (pCommitter->skmRow.uid == uid && sver == pCommitter->skmRow.pTSchema->version) goto _exit; + } else { + if (sver == pCommitter->skmRow.pTSchema->version) goto _exit; + } + } + } + + pCommitter->skmRow.suid = suid; + pCommitter->skmRow.uid = uid; + tTSchemaDestroy(pCommitter->skmRow.pTSchema); + code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); + if (code) { + goto _exit; + } + +_exit: + return code; +} + static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { int32_t code = 0; @@ -308,8 +355,17 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { } else { pCommitter->dReader.iBlockL++; if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - pCommitter->dReader.pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - code = tsdbReadLastBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockL, pBlockDatal); + SBlockL *pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); + int64_t suid = pBlockL->suid; + int64_t uid = pBlockL->maxUid; + + code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid, 1 /*TODO*/); + if (code) goto _exit; + + code = tBlockDataInit(pBlockDatal, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); + if (code) goto _exit; + + code = tsdbReadLastBlock(pCommitter->dReader.pReader, pBlockL, pBlockDatal); if (code) goto _exit; pCommitter->dReader.iRow = 0; @@ -444,54 +500,6 @@ _err: return code; } -static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { - int32_t code = 0; - - if (pCommitter->skmTable.pTSchema) { - if (pCommitter->skmTable.suid == suid) { - if (suid == 0) { - if (pCommitter->skmTable.uid == uid && sver == pCommitter->skmTable.pTSchema->version) goto _exit; - } else { - if (sver == pCommitter->skmTable.pTSchema->version) goto _exit; - } - } - } - - pCommitter->skmTable.suid = suid; - pCommitter->skmTable.uid = uid; - tTSchemaDestroy(pCommitter->skmTable.pTSchema); - code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmTable.pTSchema); - if (code) goto _exit; - -_exit: - return code; -} - -static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { - int32_t code = 0; - - if (pCommitter->skmRow.pTSchema) { - if (pCommitter->skmRow.suid == suid) { - if (suid == 0) { - if (pCommitter->skmRow.uid == uid && sver == pCommitter->skmRow.pTSchema->version) goto _exit; - } else { - if (sver == pCommitter->skmRow.pTSchema->version) goto _exit; - } - } - } - - pCommitter->skmRow.suid = suid; - pCommitter->skmRow.uid = uid; - tTSchemaDestroy(pCommitter->skmRow.pTSchema); - code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); - if (code) { - goto _exit; - } - -_exit: - return code; -} - static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { int32_t code = 0; SBlockData *pBlockData = &pCommitter->dWriter.bData; @@ -779,13 +787,13 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { STbData *pTbData = pIter->pTbData; int32_t nRow = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); - if (pCommitter->dReader.pRowInfo) { - if (pCommitter->dReader.pRowInfo->suid) { + if (pCommitter->dReader.pRowInfo && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pRowInfo) == 0) { + if (pCommitter->dReader.pRowInfo->suid) { // super table for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break; nRow++; } - } else { + } else { // normal table ASSERT(pCommitter->dReader.iRow == 0); nRow += pCommitter->dReader.bDatal.nRow; } @@ -812,10 +820,13 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { pBlockData = &pCommitter->dWriter.bDatal; // commit and reset block data schema if need - if (pBlockData->nRow > 0) { - if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; + // QUESTION: Is there a case that pBlockData->nRow == 0 but need to change schema ? + if (pBlockData->suid || pBlockData->uid) { + if (pBlockData->suid != pTbData->uid || pBlockData->suid == 0) { + if (pBlockData->nRow > 0) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } tBlockDataReset(pBlockData); } @@ -962,10 +973,12 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { pBlock = NULL; } - code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid, pTbData->maxSkmVer); + code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid, pTbData->maxSkmVer /*TODO*/); if (code) goto _err; tMapDataReset(&pCommitter->dWriter.mBlock); + code = tBlockDataInit(&pCommitter->dReader.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); + if (code) goto _err; code = tBlockDataInit(&pCommitter->dWriter.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); if (code) goto _err; @@ -1130,11 +1143,12 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { ASSERT(pBlockDataR->nRow > 0); // commit and reset block data schema if need - if (pBlockDataW->nRow > 0) { - if (pBlockDataW->suid != pCommitter->dReader.pRowInfo->suid || pBlockDataW->suid == 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - + if (pBlockDataW->suid || pBlockDataW->uid) { + if (pBlockDataW->suid != suid || pBlockDataW->suid == 0) { + if (pBlockDataW->nRow > 0) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } tBlockDataReset(pBlockDataW); } } @@ -1151,9 +1165,10 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { // check if it can make sure that one table data in one block int32_t nRow = 0; if (pBlockDataR->suid) { - for (int32_t iRow = pCommitter->dReader.iRow; (iRow < pBlockDataR->nRow) && (pBlockDataR->aUid[iRow] == uid); - iRow++) { + int32_t iRow = pCommitter->dReader.iRow; + while ((iRow < pBlockDataR->nRow) && (pBlockDataR->aUid[iRow] == uid)) { nRow++; + iRow++; } } else { ASSERT(pCommitter->dReader.iRow == 0); From 0e06d86f7bac208b03d456653eae05e513a9a1dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 11:40:49 +0000 Subject: [PATCH 034/194] fix some bug --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 5 ++--- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0e10d1b000..f46726e647 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -187,7 +187,7 @@ int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, ui int32_t *szOut, uint8_t **ppBuf); int32_t tsdbDecmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t szOut, uint8_t **ppBuf); -int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int32_t nOut, uint8_t **ppBuf); int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, uint8_t **ppBuf); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index f73883d050..af1342daf3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1050,11 +1050,10 @@ _err: int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { int32_t code = 0; - STsdb *pTsdb = (*ppWriter)->pTsdb; + STsdb *pTsdb = NULL; if (*ppWriter == NULL) goto _exit; - - if (sync) { + pTsdb = (*ppWriter)->pTsdb if (sync) { if (taosFsyncFile((*ppWriter)->pHeadFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d0d354e467..d5bd1f8c0c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1794,7 +1794,7 @@ _exit: return code; } -int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int8_t nOut, +int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int32_t nOut, uint8_t **ppBuf) { int32_t code = 0; From 50d03237ed07110b841b46fb8f1bd2f96de87163 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 12:14:16 +0000 Subject: [PATCH 035/194] fix more bug --- source/dnode/vnode/src/tsdb/tsdbFS.c | 17 ++++------------- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 7 +++++-- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 74f1aef1fc..247de99338 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -576,10 +576,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - fSet.pHeadF->nRef = 0; - fSet.pHeadF->commitID = pSet->pHeadF->commitID; - fSet.pHeadF->size = pSet->pHeadF->size; - fSet.pHeadF->offset = pSet->pHeadF->offset; + *fSet.pHeadF = *pSet->pHeadF; // data fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); @@ -587,9 +584,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - fSet.pDataF->nRef = 0; - fSet.pDataF->commitID = pSet->pDataF->commitID; - fSet.pDataF->size = pSet->pDataF->size; + *fSet.pDataF = *pSet->pDataF; // data fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); @@ -597,9 +592,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - fSet.pLastF->nRef = 0; - fSet.pLastF->commitID = pSet->pLastF->commitID; - fSet.pLastF->size = pSet->pLastF->size; + *fSet.pLastF = *pSet->pLastF; // last fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); @@ -607,9 +600,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - fSet.pSmaF->nRef = 0; - fSet.pSmaF->commitID = pSet->pSmaF->commitID; - fSet.pSmaF->size = pSet->pSmaF->size; + *fSet.pSmaF = *pSet->pSmaF; if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index af1342daf3..06aa5ef3cb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -63,6 +63,7 @@ _err: int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { int32_t code = 0; SDelFWriter *pWriter = *ppWriter; + STsdb *pTsdb = pWriter->pTsdb; // sync if (sync && taosFsyncFile(pWriter->pWriteH) < 0) { @@ -83,7 +84,7 @@ int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { return code; _err: - tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -1053,7 +1054,9 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { STsdb *pTsdb = NULL; if (*ppWriter == NULL) goto _exit; - pTsdb = (*ppWriter)->pTsdb if (sync) { + + pTsdb = (*ppWriter)->pTsdb; + if (sync) { if (taosFsyncFile((*ppWriter)->pHeadFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; From 78985ce4f1cca997c19d932ffce04d2ac7d99d75 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 13:05:21 +0000 Subject: [PATCH 036/194] fix more bug --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 ++++++ source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 13 +++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 2395a4cd8b..5d30599881 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1222,6 +1222,12 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { // commit current table data code = tsdbCommitTableData(pCommitter, pTbData); if (code) goto _err; + + // move next reader table data if need + if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { + code = tsdbCommitterNextTableData(pCommitter); + if (code) goto _err; + } } code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 06aa5ef3cb..fbdcd0f053 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1237,6 +1237,8 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { pHeadFile->size += size; _exit: + tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", TD_VID(pWriter->pTsdb->pVnode), + pHeadFile->offset, size, taosArrayGetSize(aBlockIdx)); return code; _err: @@ -1277,8 +1279,10 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBloc pBlockIdx->size = size; pHeadFile->size += size; - tsdbTrace("vgId:%d, write block, offset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), pBlockIdx->offset, - pBlockIdx->size); + tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 + " size:%" PRId64 " nItem:%d", + TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, + pBlockIdx->offset, pBlockIdx->size, mBlock->nItem); return code; _err: @@ -1331,6 +1335,8 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { pHeadFile->size += size; _exit: + tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), + pHeadFile->loffset, size); return code; _err: @@ -1542,6 +1548,9 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } _exit: + tsdbTrace("vgId:%d tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d", + TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset, + pBlkInfo->szBlock); return code; _err: From 4ac14697b781a83292bb1fdd428306bb1a48ee97 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Aug 2022 13:23:50 +0000 Subject: [PATCH 037/194] fix more bug --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 5d30599881..ae7c5f1a90 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -822,7 +822,7 @@ static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { // commit and reset block data schema if need // QUESTION: Is there a case that pBlockData->nRow == 0 but need to change schema ? if (pBlockData->suid || pBlockData->uid) { - if (pBlockData->suid != pTbData->uid || pBlockData->suid == 0) { + if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { if (pBlockData->nRow > 0) { code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index fbdcd0f053..69a4227486 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -771,7 +771,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (code) goto _err; p += hdr.szKey; - ASSERT(p - pReader->pBuf1 == sizeof(TSCKSUM)); + ASSERT(p - pReader->pBuf1 == pBlkInfo->szKey - sizeof(TSCKSUM)); // read and decode columns if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit; @@ -1195,7 +1195,7 @@ _err: int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { int32_t code = 0; SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size; + int64_t size = 0; int64_t n; // check From ae7550eb8639ac616d86ff5dcee5a7d23d20ad5b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 12 Aug 2022 21:00:48 +0800 Subject: [PATCH 038/194] enh: rsma exec in async mode --- include/common/tmsg.h | 4 + source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/inc/sma.h | 12 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/sma/smaEnv.c | 5 +- source/dnode/vnode/src/sma/smaRollup.c | 80 +++++++++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 26 ---- source/dnode/vnode/src/vnd/vnodeQuery.c | 19 +++ source/dnode/vnode/src/vnd/vnodeSvr.c | 4 +- source/libs/executor/inc/executil.h | 3 +- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/inc/tsimplehash.h | 42 ++++-- source/libs/executor/src/executil.c | 9 +- source/libs/executor/src/executorimpl.c | 22 +-- source/libs/executor/src/scanoperator.c | 4 +- source/libs/executor/src/timewindowoperator.c | 54 ++++---- source/libs/executor/src/tsimplehash.c | 126 ++++++++++++------ .../libs/executor/test/tSimpleHashTests.cpp | 14 +- 18 files changed, 273 insertions(+), 156 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index df127af256..24092c7e44 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2658,6 +2658,10 @@ typedef struct { SEpSet epSet; } SVgEpSet; +typedef struct { + int32_t padding; +} SRSmaExecMsg; + typedef struct { int64_t suid; int8_t level; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 18a7583f4c..e93a62b27b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -63,6 +63,7 @@ void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); +int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray* list); void *vnodeGetIdx(SVnode *pVnode); void *vnodeGetIvtIdx(SVnode *pVnode); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 944d7759b2..fb8352d543 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -122,17 +122,19 @@ struct SRSmaInfoItem { }; struct SRSmaInfo { - STSchema *pTSchema; - int64_t suid; - int64_t refId; // refId of SRSmaStat - int8_t delFlag; + STSchema *pTSchema; + STaosQueue *queue; // buffer queue of SubmitReq + STaosQall *qall; + int64_t suid; + int64_t refId; // refId of SRSmaStat + int8_t delFlag; T_REF_DECLARE() SRSmaInfoItem items[TSDB_RETENTION_L2]; void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t void *iTaskInfo[TSDB_RETENTION_L2]; // immutable }; -#define RSMA_INFO_HEAD_LEN 32 +#define RSMA_INFO_HEAD_LEN offsetof(SRSmaInfo, items) #define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1) #define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1) #define RSMA_INFO_QTASK(r, i) ((r)->taskInfo[i]) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 43bb92ec23..513f4da33c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -169,7 +169,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); -int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list); + SSubmitReq* tdBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, const char* stbFullName, int32_t vgId, SBatchDeleteReq* pDeleteReq); diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index ccb6ad3a72..50db1123ae 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -264,8 +264,6 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED); // step 2: destroy the rsma info and associated fetch tasks - // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. -#if 1 if (taosHashGetSize(RSMA_INFO_HASH(pStat)) > 0) { void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL); while (infoHash) { @@ -274,7 +272,6 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash); } } -#endif taosHashCleanup(RSMA_INFO_HASH(pStat)); // step 3: wait all triggered fetch tasks finished @@ -292,7 +289,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { nLoops = 0; } } - + // step 4: free pStat taosMemoryFreeClear(pStat); } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index b7a2efd489..1660223d77 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -139,6 +139,14 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { if (isDeepFree) { taosMemoryFreeClear(pInfo->pTSchema); } + + if (isDeepFree) { + if (pInfo->queue) taosCloseQueue(pInfo->queue); + if (pInfo->qall) taosFreeQall(pInfo->qall); + pInfo->queue = NULL; + pInfo->qall = NULL; + } + taosMemoryFree(pInfo); } @@ -179,7 +187,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pRSmaInfo->taskInfo[i]) { - if ((qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, true) < 0)) { + if (((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, true)) < 0)) { tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, terrstr()); @@ -351,6 +359,12 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con goto _err; } pRSmaInfo->pTSchema = pTSchema; + if (!(pRSmaInfo->queue = taosOpenQueue())) { + goto _err; + } + if (!(pRSmaInfo->qall = taosAllocateQall())) { + goto _err; + } pRSmaInfo->suid = suid; pRSmaInfo->refId = RSMA_REF_ID(pStat); T_REF_INIT_VAL(pRSmaInfo, 1); @@ -615,7 +629,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm while (1) { uint64_t ts; int32_t code = qExecTaskOpt(taskInfo, pResList, &ts); - if (code < 0) { + if (code < 0) { if (code == TSDB_CODE_QRY_IN_EXEC) { break; } else { @@ -662,10 +676,9 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm goto _err; } taosMemoryFreeClear(pReq); - + smaDebug("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " version:%" PRIi64, SMA_VID(pSma), suid, pItem->level, output->info.version); - } } @@ -841,30 +854,45 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; } +/** + * @brief retrieve rsma meta and init + * + * @param pSma + * @param nTables number of tables of rsma + * @return int32_t + */ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { - SVnode *pVnode = pSma->pVnode; + SVnode *pVnode = pSma->pVnode; + SArray *suidList = NULL; + STbUidStore uidStore = {0}; + SMetaReader mr = {0}; - SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t)); - if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) { - taosArrayDestroy(suidList); + if (!(suidList = taosArrayInit(1, sizeof(tb_uid_t)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + if (vnodeGetStbIdList(pSma->pVnode, 0, suidList) < 0) { smaError("vgId:%d, failed to restore rsma env since get stb id list error: %s", TD_VID(pVnode), terrstr()); - return TSDB_CODE_FAILED; + goto _err; } int64_t arrSize = taosArrayGetSize(suidList); - if (nTables) { - *nTables = arrSize; - } - if (arrSize == 0) { + if (nTables) { + *nTables = 0; + } taosArrayDestroy(suidList); smaDebug("vgId:%d, no need to restore rsma env since empty stb id list", TD_VID(pVnode)); return TSDB_CODE_SUCCESS; } - SMetaReader mr = {0}; + int64_t nRsmaTables = 0; metaReaderInit(&mr, SMA_META(pSma), 0); + if (!(uidStore.tbUids = taosArrayInit(1024, sizeof(tb_uid_t)))) { + goto _err; + } for (int64_t i = 0; i < arrSize; ++i) { tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid); @@ -877,6 +905,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { ASSERT(mr.me.type == TSDB_SUPER_TABLE); ASSERT(mr.me.uid == suid); if (TABLE_IS_ROLLUP(mr.me.flags)) { + ++nRsmaTables; SRSmaParam *param = &mr.me.stbEntry.rsmaParam; for (int i = 0; i < TSDB_RETENTION_L2; ++i) { smaDebug("vgId:%d, rsma restore, table:%" PRIi64 " level:%d, maxdelay:%" PRIi64 " watermark:%" PRIi64 @@ -887,17 +916,40 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { smaError("vgId:%d, rsma restore env failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); goto _err; } + + // reload all ctbUids for suid + uidStore.suid = suid; + if (vnodeGetCtbIdList(pVnode, suid, uidStore.tbUids) < 0) { + smaError("vgId:%d, rsma restore, get ctb idlist failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, + terrstr()); + goto _err; + } + + if (tdUpdateTbUidList(pVnode->pSma, &uidStore) < 0) { + smaError("vgId:%d, rsma restore, update tb uid list failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, + terrstr()); + goto _err; + } + + taosArrayClear(uidStore.tbUids); + smaDebug("vgId:%d, rsma restore env success for %" PRIi64, TD_VID(pVnode), suid); } } metaReaderClear(&mr); taosArrayDestroy(suidList); + tdUidStoreDestory(&uidStore); + + if (nTables) { + *nTables = nRsmaTables; + } return TSDB_CODE_SUCCESS; _err: metaReaderClear(&mr); taosArrayDestroy(suidList); + tdUidStoreDestory(&uidStore); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0831f3d75a..335b311d00 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2585,32 +2585,6 @@ void* tsdbGetIvtIdx(SMeta* pMeta) { uint64_t getReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; } -/** - * @brief Get all suids since suid - * - * @param pMeta - * @param suid return all suids in one vnode if suid is 0 - * @param list - * @return int32_t - */ -int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list) { - SMStbCursor* pCur = metaOpenStbCursor(pMeta, suid); - if (!pCur) { - return TSDB_CODE_FAILED; - } - - while (1) { - tb_uid_t id = metaStbCursorNext(pCur); - if (id == 0) { - break; - } - - taosArrayPush(list, &id); - } - - metaCloseStbCursor(pCur); - return TSDB_CODE_SUCCESS; -} // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTableList, STsdbReader** ppReader, diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index d55f1796ad..8d799e919d 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -424,6 +424,25 @@ int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list) { return TSDB_CODE_SUCCESS; } +int32_t vnodeGetStbIdList(SVnode* pVnode, int64_t suid, SArray* list) { + SMStbCursor* pCur = metaOpenStbCursor(pVnode->pMeta, suid); + if (!pCur) { + return TSDB_CODE_FAILED; + } + + while (1) { + tb_uid_t id = metaStbCursorNext(pCur); + if (id == 0) { + break; + } + + taosArrayPush(list, &id); + } + + metaCloseStbCursor(pCur); + return TSDB_CODE_SUCCESS; +} + int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid); if (!pCur) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d5c5e18668..3a25933ec4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -522,7 +522,9 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR } tqUpdateTbUidList(pVnode->pTq, tbUids, true); - tdUpdateTbUidList(pVnode->pSma, pStore); + if (tdUpdateTbUidList(pVnode->pSma, pStore) < 0) { + goto _exit; + } tdUidStoreFree(pStore); // prepare rsp diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 52c73f85f5..58b2c1b095 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -22,6 +22,7 @@ #include "tbuffer.h" #include "tcommon.h" #include "tpagedbuf.h" +#include "tsimplehash.h" #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ do { \ @@ -102,7 +103,7 @@ static FORCE_INLINE void setResultBufPageDirty(SDiskbasedBuf* pBuf, SResultRowPo setBufPageDirty(pPage, true); } -void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t order); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 410bab341c..b7b0d5123e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -296,7 +296,7 @@ enum { }; typedef struct SAggSupporter { - SHashObj* pResultRowHashTable; // quick locate the window object for each result + SSHashObj* pResultRowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row diff --git a/source/libs/executor/inc/tsimplehash.h b/source/libs/executor/inc/tsimplehash.h index a56f8e8c04..4c5a80e2f1 100644 --- a/source/libs/executor/inc/tsimplehash.h +++ b/source/libs/executor/inc/tsimplehash.h @@ -17,7 +17,6 @@ #define TDENGINE_TSIMPLEHASH_H #include "tarray.h" -#include "tlockfree.h" #ifdef __cplusplus extern "C" { @@ -27,6 +26,10 @@ typedef uint32_t (*_hash_fn_t)(const char *, uint32_t); typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len); typedef void (*_hash_free_fn_t)(void *); +/** + * @brief single thread hash + * + */ typedef struct SSHashObj SSHashObj; /** @@ -36,7 +39,7 @@ typedef struct SSHashObj SSHashObj; * @param fn hash function to generate the hash value * @return */ -SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen); +SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn); /** * return the size of hash table @@ -48,22 +51,26 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj); int32_t tSimpleHashPrint(const SSHashObj *pHashObj); /** - * put element into hash table, if the element with the same key exists, update it - * @param pHashObj - * @param key - * @param data - * @return + * @brief put element into hash table, if the element with the same key exists, update it + * + * @param pHashObj + * @param key + * @param keyLen + * @param data + * @param dataLen + * @return int32_t */ -int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data); +int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t dataLen); /** * return the payload data with the specified key * * @param pHashObj * @param key + * @param keyLen * @return */ -void *tSimpleHashGet(SSHashObj *pHashObj, const void *key); +void *tSimpleHashGet(SSHashObj *pHashObj, const void *key, size_t keyLen); /** * remove item with the specified key @@ -71,7 +78,7 @@ void *tSimpleHashGet(SSHashObj *pHashObj, const void *key); * @param key * @param keyLen */ -int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key); +int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key, size_t keyLen); /** * Clear the hash table. @@ -98,7 +105,7 @@ size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj); * @param keyLen * @return */ -void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen); +void *tSimpleHashGetKey(void *data, size_t* keyLen); /** * Create the hash table iterator @@ -109,7 +116,18 @@ void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen); */ void *tSimpleHashIterate(const SSHashObj *pHashObj, void *data, int32_t *iter); +/** + * Create the hash table iterator + * + * @param pHashObj + * @param data + * @param key + * @param iter + * @return void* + */ +void *tSimpleHashIterateKV(const SSHashObj *pHashObj, void *data, void **key, int32_t *iter); + #ifdef __cplusplus } #endif -#endif // TDENGINE_TSIMPLEHASH_H +#endif // TDENGINE_TSIMPLEHASH_H \ No newline at end of file diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 34247d3b47..2e6bd312f3 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -97,7 +97,7 @@ int32_t resultrowComparAsc(const void* p1, const void* p2) { static int32_t resultrowComparDesc(const void* p1, const void* p2) { return resultrowComparAsc(p2, p1); } -void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order) { +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t order) { if (pGroupResInfo->pRows != NULL) { taosArrayDestroy(pGroupResInfo->pRows); } @@ -106,9 +106,10 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int void* pData = NULL; pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES); - size_t keyLen = 0; - while ((pData = taosHashIterate(pHashmap, pData)) != NULL) { - void* key = taosHashGetKey(pData, &keyLen); + size_t keyLen = 0; + int32_t iter = 0; + while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { + void* key = tSimpleHashGetKey(pData, &keyLen); SResKeyPos* p = taosMemoryMalloc(keyLen + sizeof(SResultRowPosition)); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index be129cb6b4..4709d1818c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -250,7 +250,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); SResultRow* pResult = NULL; @@ -292,7 +292,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // add a new result set for a new group SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; - taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, + tSimpleHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, sizeof(SResultRowPosition)); } @@ -301,7 +301,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // too many time window in query if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH && - taosHashGetSize(pSup->pResultRowHashTable) > MAX_INTERVAL_TIME_WINDOW) { + tSimpleHashGetSize(pSup->pResultRowHashTable) > MAX_INTERVAL_TIME_WINDOW) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } @@ -3013,7 +3013,7 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len } SOptrBasicInfo* pInfo = (SOptrBasicInfo*)(pOperator->info); SAggSupporter* pSup = (SAggSupporter*)POINTER_SHIFT(pOperator->info, sizeof(SOptrBasicInfo)); - int32_t size = taosHashGetSize(pSup->pResultRowHashTable); + int32_t size = tSimpleHashGetSize(pSup->pResultRowHashTable); size_t keyLen = sizeof(uint64_t) * 2; // estimate the key length int32_t totalSize = sizeof(int32_t) + sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); @@ -3041,9 +3041,10 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len setBufPageDirty(pPage, true); releaseBufPage(pSup->pResultBuf, pPage); - void* pIter = taosHashIterate(pSup->pResultRowHashTable, NULL); + int32_t iter = 0; + void* pIter = tSimpleHashIterate(pSup->pResultRowHashTable, NULL, &iter); while (pIter) { - void* key = taosHashGetKey(pIter, &keyLen); + void* key = tSimpleHashGetKey(pIter, &keyLen); SResultRowPosition* p1 = (SResultRowPosition*)pIter; pPage = (SFilePage*)getBufPage(pSup->pResultBuf, p1->pageId); @@ -3075,7 +3076,7 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len memcpy(*result + offset, pRow, pSup->resultRowSize); offset += pSup->resultRowSize; - pIter = taosHashIterate(pSup->pResultRowHashTable, pIter); + pIter = tSimpleHashIterate(pSup->pResultRowHashTable, pIter, &iter); } *(int32_t*)(*result) = offset; @@ -3110,7 +3111,7 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) { // add a new result set for a new group SResultRowPosition pos = {.pageId = resultRow->pageId, .offset = resultRow->offset}; - taosHashPut(pSup->pResultRowHashTable, result + offset, keyLen, &pos, sizeof(SResultRowPosition)); + tSimpleHashPut(pSup->pResultRowHashTable, result + offset, keyLen, &pos, sizeof(SResultRowPosition)); offset += keyLen; int32_t valueLen = *(int32_t*)(result + offset); @@ -3407,7 +3408,8 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); - pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); + // pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); + pAggSup->pResultRowHashTable = tSimpleHashInit(100000, hashFn); if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3433,7 +3435,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n void cleanupAggSup(SAggSupporter* pAggSup) { taosMemoryFreeClear(pAggSup->keyBuf); - taosHashCleanup(pAggSup->pResultRowHashTable); + tSimpleHashCleanup(pAggSup->pResultRowHashTable); destroyDiskbasedBuf(pAggSup->pResultBuf); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d8de8df163..c404fca597 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -178,8 +178,8 @@ static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t gro STableScanInfo* pTableScanInfo = pOperator->info; - SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pTableScanInfo->pdInfo.pAggSup->pResultRowHashTable, buf, GET_RES_WINDOW_KEY_LEN(sizeof(groupId))); + SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pTableScanInfo->pdInfo.pAggSup->pResultRowHashTable, buf, + GET_RES_WINDOW_KEY_LEN(sizeof(groupId))); if (p1 == NULL) { return NULL; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 6778e97d7a..abc1a76d74 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1383,7 +1383,7 @@ bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t int32_t numOfOutput) { SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (!p1) { // window has been closed return false; @@ -1396,14 +1396,14 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) size_t bytes = sizeof(TSKEY); SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (!p1) { // window has been closed return false; } // SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId); // dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage); - taosHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + tSimpleHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); return true; } @@ -1453,11 +1453,12 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* } } -static int32_t getAllIntervalWindow(SHashObj* pHashMap, SHashObj* resWins) { +static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { void* pIte = NULL; size_t keyLen = 0; - while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { - void* key = taosHashGetKey(pIte, &keyLen); + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { + void* key = tSimpleHashGetKey(pIte, &keyLen); uint64_t groupId = *(uint64_t*)key; ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); @@ -1470,16 +1471,18 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SHashObj* resWins) { return TSDB_CODE_SUCCESS; } -static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, +static int32_t closeIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pRecyPages, SDiskbasedBuf* pDiscBuf) { qDebug("===stream===close interval window"); void* pIte = NULL; - size_t keyLen = 0; - while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { - void* key = taosHashGetKey(pIte, &keyLen); + void* key = NULL; + size_t keyLen = GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)); + int32_t iter = 0; + while ((pIte = tSimpleHashIterateKV(pHashMap, pIte, &key, &iter)) != NULL) { + // void* key = tSimpleHashGetKey(pIte, &keyLen); uint64_t groupId = *(uint64_t*)key; - ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); + // ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); STimeWindow win; win.skey = ts; @@ -1515,7 +1518,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, } char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); - taosHashRemove(pHashMap, keyBuf, keyLen); + tSimpleHashRemove(pHashMap, keyBuf, keyLen); } } return TSDB_CODE_SUCCESS; @@ -2853,7 +2856,7 @@ bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) { int32_t bytes = sizeof(TSKEY); SET_RES_WINDOW_KEY(pSup->keyBuf, &ts, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); return p1 != NULL; } @@ -2894,8 +2897,9 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId); - SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); + SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, + GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); + return p1 == NULL; } @@ -3023,7 +3027,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc } static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) { - taosHashClear(pInfo->aggSup.pResultRowHashTable); + tSimpleHashClear(pInfo->aggSup.pResultRowHashTable); clearDiskbasedBuf(pInfo->aggSup.pResultBuf); cleanupResultRowInfo(&pInfo->binfo.resultRowInfo); initResultRowInfo(&pInfo->binfo.resultRowInfo); @@ -4938,14 +4942,14 @@ static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, ui SExprSupp* pSup = &pOperatorInfo->exprSupp; SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId); - SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, - GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet( + iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); ASSERT(p1 != NULL); finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pSup->pCtx, pSup->pExprInfo, pSup->numOfExprs, pSup->rowEntryInfoOffset, pResultBlock, pTaskInfo); - taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); - ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); + tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); return TSDB_CODE_SUCCESS; } @@ -4968,7 +4972,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR // there is an result exists if (miaInfo->curTs != INT64_MIN) { - ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); + ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); if (ts != miaInfo->curTs) { outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs); @@ -4976,7 +4980,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR } } else { miaInfo->curTs = ts; - ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); + ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); } STimeWindow win = {0}; @@ -5053,7 +5057,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { if (pBlock == NULL) { // close last unfinalized time window if (miaInfo->curTs != INT64_MIN) { - ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); + ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); miaInfo->curTs = INT64_MIN; } @@ -5231,12 +5235,12 @@ static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t table SExprSupp* pExprSup = &pOperatorInfo->exprSupp; SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId); - SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, + SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); ASSERT(p1 != NULL); finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo); - taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 7989ad2b5a..dbb50f958d 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -14,7 +14,6 @@ */ #include "tsimplehash.h" -#include "os.h" #include "taoserror.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 @@ -31,19 +30,21 @@ taosMemoryFreeClear(_n); \ } while (0); +#pragma pack(push, 4) typedef struct SHNode { struct SHNode *next; + uint32_t keyLen : 20; + uint32_t dataLen : 12; char data[]; } SHNode; +#pragma pack(pop) struct SSHashObj { SHNode **hashList; size_t capacity; // number of slots - int64_t size; // number of elements in hash table - _hash_fn_t hashFp; // hash function - _equal_fn_t equalFp; // equal function - int32_t keyLen; - int32_t dataLen; + int64_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function }; static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { @@ -54,7 +55,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { return i; } -SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen) { +SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn) { ASSERT(fn != NULL); if (capacity == 0) { @@ -74,8 +75,6 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t pHashObj->hashFp = fn; ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); - pHashObj->keyLen = keyLen; - pHashObj->dataLen = dataLen; pHashObj->hashList = (SHNode **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); if (!pHashObj->hashList) { @@ -93,16 +92,17 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { return (int32_t)atomic_load_64((int64_t *)&pHashObj->size); } -static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHNode *pNewNode = taosMemoryMalloc(sizeof(SHNode) + keyLen + dsize); +static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *data, size_t dataLen, uint32_t hashVal) { + SHNode *pNewNode = taosMemoryMalloc(sizeof(SHNode) + keyLen + dataLen); if (!pNewNode) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - + pNewNode->keyLen = keyLen; + pNewNode->dataLen = dataLen; pNewNode->next = NULL; - memcpy(GET_SHASH_NODE_DATA(pNewNode), pData, dsize); - memcpy(GET_SHASH_NODE_KEY(pNewNode, dsize), key, keyLen); + memcpy(GET_SHASH_NODE_DATA(pNewNode), data, dataLen); + memcpy(GET_SHASH_NODE_KEY(pNewNode, dataLen), key, keyLen); return pNewNode; } @@ -141,8 +141,8 @@ static void taosHashTableResize(SSHashObj *pHashObj) { SHNode *pPrev = NULL; while (pNode != NULL) { - void *key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen); - uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen); + void *key = GET_SHASH_NODE_KEY(pNode, pNode->dataLen); + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pNode->keyLen); int32_t newIdx = HASH_INDEX(hashVal, pHashObj->capacity); pNext = pNode->next; @@ -170,12 +170,12 @@ static void taosHashTableResize(SSHashObj *pHashObj) { // ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } -int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { +int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t dataLen) { if (!pHashObj || !key) { return -1; } - uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen); + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // need the resize process, write lock applied if (SHASH_NEED_RESIZE(pHashObj)) { @@ -186,7 +186,7 @@ int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { SHNode *pNode = pHashObj->hashList[slot]; if (!pNode) { - SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->dataLen, hashVal); + SHNode *pNewNode = doCreateHashNode(key, keyLen, data, dataLen, hashVal); if (!pNewNode) { return -1; } @@ -197,14 +197,14 @@ int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { } while (pNode) { - if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) { + if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pNode->dataLen), key, keyLen) == 0) { break; } pNode = pNode->next; } if (!pNode) { - SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->dataLen, hashVal); + SHNode *pNewNode = doCreateHashNode(key, keyLen, data, dataLen, hashVal); if (!pNewNode) { return -1; } @@ -212,16 +212,16 @@ int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { pHashObj->hashList[slot] = pNewNode; atomic_add_fetch_64(&pHashObj->size, 1); } else { // update data - memcpy(GET_SHASH_NODE_DATA(pNode), data, pHashObj->dataLen); + memcpy(GET_SHASH_NODE_DATA(pNode), data, dataLen); } return 0; } -static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void *key, int32_t index) { +static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void *key, size_t keyLen, int32_t index) { SHNode *pNode = pHashObj->hashList[index]; while (pNode) { - if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) { + if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pNode->dataLen), key, keyLen) == 0) { break; } @@ -233,12 +233,12 @@ static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) { return tSimpleHashGetSize(pHashObj) == 0; } -void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) { +void *tSimpleHashGet(SSHashObj *pHashObj, const void *key, size_t keyLen) { if (!pHashObj || taosHashTableEmpty(pHashObj) || !key) { return NULL; } - uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen); + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHNode *pNode = pHashObj->hashList[slot]; @@ -247,7 +247,7 @@ void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) { } char *data = NULL; - pNode = doSearchInEntryList(pHashObj, key, slot); + pNode = doSearchInEntryList(pHashObj, key, keyLen, slot); if (pNode != NULL) { data = GET_SHASH_NODE_DATA(pNode); } @@ -255,19 +255,19 @@ void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) { return data; } -int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key) { +int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key, size_t keyLen) { if (!pHashObj || !key) { return TSDB_CODE_FAILED; } - uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen); + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHNode *pNode = pHashObj->hashList[slot]; SHNode *pPrev = NULL; while (pNode) { - if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) { + if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pNode->dataLen), key, keyLen) == 0) { if (!pPrev) { pHashObj->hashList[slot] = pNode->next; } else { @@ -312,6 +312,7 @@ void tSimpleHashCleanup(SSHashObj *pHashObj) { tSimpleHashClear(pHashObj); taosMemoryFreeClear(pHashObj->hashList); + taosMemoryFree(pHashObj); } size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { @@ -322,23 +323,13 @@ size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { return (pHashObj->capacity * sizeof(void *)) + sizeof(SHNode) * tSimpleHashGetSize(pHashObj) + sizeof(SSHashObj); } -void *tSimpleHashGetKey(const SSHashObj *pHashObj, void *data, size_t *keyLen) { -#if 0 - int32_t offset = offsetof(SHNode, data); - SHNode *node = ((SHNode *)(char *)data - offset); +void *tSimpleHashGetKey(void *data, size_t *keyLen) { + SHNode *node = (SHNode *)((char *)data - offsetof(SHNode, data)); if (keyLen) { - *keyLen = pHashObj->keyLen; + *keyLen = node->keyLen; } - return POINTER_SHIFT(data, pHashObj->dataLen); - - return GET_SHASH_NODE_KEY(node, pHashObj->dataLen); -#endif - if (keyLen) { - *keyLen = pHashObj->keyLen; - } - - return POINTER_SHIFT(data, pHashObj->dataLen); + return POINTER_SHIFT(data, node->dataLen); } void *tSimpleHashIterate(const SSHashObj *pHashObj, void *data, int32_t *iter) { @@ -376,5 +367,52 @@ void *tSimpleHashIterate(const SSHashObj *pHashObj, void *data, int32_t *iter) { return GET_SHASH_NODE_DATA(pNode); } + return NULL; +} + +void *tSimpleHashIterateKV(const SSHashObj *pHashObj, void *data, void **key, int32_t *iter) { + if (!pHashObj) { + return NULL; + } + + SHNode *pNode = NULL; + + if (!data) { + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + pNode = pHashObj->hashList[i]; + if (!pNode) { + continue; + } + *iter = i; + if (key) { + *key = GET_SHASH_NODE_KEY(pNode, pNode->dataLen); + } + return GET_SHASH_NODE_DATA(pNode); + } + return NULL; + } + + pNode = (SHNode *)((char *)data - offsetof(SHNode, data)); + + if (pNode->next) { + if (key) { + *key = GET_SHASH_NODE_KEY(pNode->next, pNode->next->dataLen); + } + return GET_SHASH_NODE_DATA(pNode->next); + } + + ++(*iter); + for (int32_t i = *iter; i < pHashObj->capacity; ++i) { + pNode = pHashObj->hashList[i]; + if (!pNode) { + continue; + } + *iter = i; + if (key) { + *key = GET_SHASH_NODE_KEY(pNode, pNode->dataLen); + } + return GET_SHASH_NODE_DATA(pNode); + } + return NULL; } \ No newline at end of file diff --git a/source/libs/executor/test/tSimpleHashTests.cpp b/source/libs/executor/test/tSimpleHashTests.cpp index a17a7146ea..acb6d434b4 100644 --- a/source/libs/executor/test/tSimpleHashTests.cpp +++ b/source/libs/executor/test/tSimpleHashTests.cpp @@ -32,31 +32,33 @@ TEST(testCase, tSimpleHashTest) { SSHashObj *pHashObj = - tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), sizeof(int64_t), sizeof(int64_t)); + tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); assert(pHashObj != nullptr); ASSERT_EQ(0, tSimpleHashGetSize(pHashObj)); + size_t keyLen = sizeof(int64_t); + size_t dataLen = sizeof(int64_t); + int64_t originKeySum = 0; for (int64_t i = 1; i <= 100; ++i) { originKeySum += i; - tSimpleHashPut(pHashObj, (const void *)&i, (const void *)&i); + tSimpleHashPut(pHashObj, (const void *)&i, keyLen, (const void *)&i, dataLen); ASSERT_EQ(i, tSimpleHashGetSize(pHashObj)); } for (int64_t i = 1; i <= 100; ++i) { - void *data = tSimpleHashGet(pHashObj, (const void *)&i); + void *data = tSimpleHashGet(pHashObj, (const void *)&i, keyLen); ASSERT_EQ(i, *(int64_t *)data); } - void *data = NULL; int32_t iter = 0; int64_t keySum = 0; int64_t dataSum = 0; while ((data = tSimpleHashIterate(pHashObj, data, &iter))) { - void *key = tSimpleHashGetKey(pHashObj, data, NULL); + void *key = tSimpleHashGetKey(data, NULL); keySum += *(int64_t *)key; dataSum += *(int64_t *)data; } @@ -65,7 +67,7 @@ TEST(testCase, tSimpleHashTest) { ASSERT_EQ(keySum, originKeySum); for (int64_t i = 1; i <= 100; ++i) { - tSimpleHashRemove(pHashObj, (const void *)&i); + tSimpleHashRemove(pHashObj, (const void *)&i, keyLen); ASSERT_EQ(100 - i, tSimpleHashGetSize(pHashObj)); } From dab6c81769d5292f9b149d6d0cc0d443e925a7d1 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 15 Aug 2022 00:16:44 +0800 Subject: [PATCH 039/194] enh: rsma batch process --- include/common/tmsg.h | 2 +- include/common/tmsgdef.h | 1 + include/util/taoserror.h | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/vnode/src/inc/sma.h | 53 +-- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/sma/smaCommit.c | 145 +++++---- source/dnode/vnode/src/sma/smaEnv.c | 4 +- source/dnode/vnode/src/sma/smaRollup.c | 339 +++++++++++++++++--- source/dnode/vnode/src/sma/smaTimeRange.c | 2 +- source/dnode/vnode/src/sma/smaUtil.c | 32 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 + source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/tsimplehash.c | 13 +- source/util/src/terror.c | 1 + 15 files changed, 425 insertions(+), 174 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 4454c061ae..a283b7c9c1 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2660,7 +2660,7 @@ typedef struct { } SVgEpSet; typedef struct { - int32_t padding; + // padding } SRSmaExecMsg; typedef struct { diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 6462c7afbf..2bf840fd01 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -201,6 +201,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) TD_DEF_MSG_TYPE(TDMT_VND_FETCH_RSMA, "vnode-fetch-rsma", SRSmaFetchMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_EXEC_RSMA, "vnode-exec-rsma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 3ca6978156..2d41874912 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -614,6 +614,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_REMOVE_EXISTS TAOS_DEF_ERROR_CODE(0, 0x3154) #define TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x3155) #define TSDB_CODE_RSMA_EMPTY_INFO TAOS_DEF_ERROR_CODE(0, 0x3156) +#define TSDB_CODE_RSMA_INVALID_SCHEMA TAOS_DEF_ERROR_CODE(0, 0x3157) //index #define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 7c6807ab87..1b4efeca7a 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -338,6 +338,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_CONTINUE, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_RSMA, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_EXEC_RSMA, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_FETCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index fb8352d543..bc204e032d 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -57,9 +57,10 @@ typedef struct { void *tmrHandle; // shared by all fetch tasks } SSmaMgmt; -#define SMA_ENV_LOCK(env) (&(env)->lock) -#define SMA_ENV_TYPE(env) ((env)->type) -#define SMA_ENV_STAT(env) ((env)->pStat) +#define SMA_ENV_LOCK(env) (&(env)->lock) +#define SMA_ENV_TYPE(env) ((env)->type) +#define SMA_ENV_STAT(env) ((env)->pStat) +#define SMA_RSMA_STAT(sma) ((SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)(sma)->pRSmaEnv)) struct STSmaStat { int8_t state; // ETsdbSmaStat @@ -86,17 +87,19 @@ struct SQTaskFWriter { }; struct SRSmaStat { - SSma *pSma; - int64_t commitAppliedVer; // vnode applied version for async commit - int64_t refId; // shared by fetch tasks - SRWLatch lock; // r/w lock for rsma fs(e.g. qtaskinfo) - int8_t triggerStat; // shared by fetch tasks - int8_t commitStat; // 0 not in committing, 1 in committing - SArray *aTaskFile; // qTaskFiles committed recently(for recovery/snapshot r/w) - SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; - SHashObj *iRsmaInfoHash; // key: stbUid, value: SRSmaInfo; immutable rsmaInfoHash + SSma *pSma; + int64_t commitAppliedVer; // vnode applied version for async commit + int64_t refId; // shared by fetch tasks + volatile int64_t qBufSize; // queue buffer size + SRWLatch lock; // r/w lock for rsma fs(e.g. qtaskinfo) + int8_t triggerStat; // shared by fetch tasks + int8_t commitStat; // 0 not in committing, 1 in committing + int8_t execStat; // 0 not in exec , 1 in exec + SArray *aTaskFile; // qTaskFiles committed recently(for recovery/snapshot r/w) + SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; }; + struct SSmaStat { union { STSmaStat tsmaStat; // time-range-wise sma @@ -105,10 +108,9 @@ struct SSmaStat { T_REF_DECLARE() }; -#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) -#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) +#define SMA_STAT_TSMA(s) (&(s)->tsmaStat) +#define SMA_STAT_RSMA(s) (&(s)->rsmaStat) #define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) -#define RSMA_IMU_INFO_HASH(r) ((r)->iRsmaInfoHash) #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) #define RSMA_COMMIT_STAT(r) (&(r)->commitStat) #define RSMA_REF_ID(r) ((r)->refId) @@ -122,23 +124,25 @@ struct SRSmaInfoItem { }; struct SRSmaInfo { - STSchema *pTSchema; - STaosQueue *queue; // buffer queue of SubmitReq - STaosQall *qall; - int64_t suid; - int64_t refId; // refId of SRSmaStat - int8_t delFlag; + STSchema *pTSchema; + int64_t suid; + int64_t refId; // refId of SRSmaStat + int8_t delFlag; T_REF_DECLARE() SRSmaInfoItem items[TSDB_RETENTION_L2]; void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t - void *iTaskInfo[TSDB_RETENTION_L2]; // immutable + STaosQueue *queue; // buffer queue of SubmitReq + STaosQall *qall; // buffer qall of SubmitReq + void *iTaskInfo[TSDB_RETENTION_L2]; // immutable qTaskInfo_t + STaosQueue *iQueue; // immutable buffer queue of SubmitReq + STaosQall *iQall; // immutable buffer qall of SubmitReq }; #define RSMA_INFO_HEAD_LEN offsetof(SRSmaInfo, items) #define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1) #define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1) #define RSMA_INFO_QTASK(r, i) ((r)->taskInfo[i]) -#define RSMA_INFO_IQTASK(r, i) ((r)->iTaskInfo[i]) +#define RSMA_INFO_IQTASK(r, i) ((r)->iTaskInfo[i]) #define RSMA_INFO_ITEM(r, i) (&(r)->items[i]) enum { @@ -230,12 +234,13 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { void tdRSmaQTaskInfoGetFileName(int32_t vid, int64_t version, char *outputName); void tdRSmaQTaskInfoGetFullName(int32_t vid, int64_t version, const char *path, char *outputName); -int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo **pDest, SRSmaInfo *pSrc); +int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); +int32_t tdRSmaProcessExecImpl(SSma *pSma); int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdProcessRSmaRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 1cc8e1bb27..f85a3d8361 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -188,6 +188,7 @@ int32_t smaAsyncCommit(SSma* pSma); int32_t smaAsyncPostCommit(SSma* pSma); int32_t smaDoRetention(SSma* pSma, int64_t now); int32_t smaProcessFetch(SSma* pSma, void* pMsg); +int32_t smaProcessExec(SSma* pSma, void* pMsg); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 373cfdfb47..807c033489 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -83,8 +83,7 @@ int32_t smaBegin(SSma *pSma) { return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); int8_t rsmaTriggerStat = atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED, TASK_TRIGGER_STAT_ACTIVE); @@ -122,8 +121,8 @@ static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SRSmaStat *pRSmaStat = SMA_STAT_RSMA(pStat); // step 1: set rsma stat paused atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); @@ -289,8 +288,7 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } - SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(SMA_ENV_STAT(pSmaEnv)); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); // cleanup outdated qtaskinfo files tdCleanupQTaskInfoFiles(pSma, pRSmaStat); @@ -314,7 +312,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + SRSmaStat *pRSmaStat = SMA_STAT_RSMA(pStat); // step 1: set rsma stat atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); @@ -336,24 +334,30 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } } - // step 3: swap rsmaInfoHash and iRsmaInfoHash + // step 3: consume the SubmitReq in buffer + if (tdRSmaProcessExecImpl(pSma) < 0) { + return TSDB_CODE_FAILED; + } + + // step 4: swap rsmaInfoHash and iRsmaInfoHash // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); ASSERT(RSMA_INFO_HASH(pRSmaStat)); - ASSERT(!RSMA_IMU_INFO_HASH(pRSmaStat)); - RSMA_IMU_INFO_HASH(pRSmaStat) = RSMA_INFO_HASH(pRSmaStat); - RSMA_INFO_HASH(pRSmaStat) = - taosHashInit(RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + void *pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); - if (!RSMA_INFO_HASH(pRSmaStat)) { - // unlock - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - smaError("vgId:%d, rsma async commit failed since %s", SMA_VID(pSma), terrstr()); - return TSDB_CODE_FAILED; + while (pIter) { + SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; + TSWAP(pInfo->iQall, pInfo->qall); + TSWAP(pInfo->iQueue, pInfo->queue); + TSWAP(pInfo->iTaskInfo[0], pInfo->taskInfo[0]); + TSWAP(pInfo->iTaskInfo[1], pInfo->taskInfo[1]); + pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter); } + atomic_store_64(&pRSmaStat->qBufSize, 0); + // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); @@ -375,11 +379,9 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); - // perform persist task for qTaskInfo - tdRSmaPersistExecImpl(pRSmaStat, RSMA_IMU_INFO_HASH(pRSmaStat)); + SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); + tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); return TSDB_CODE_SUCCESS; } @@ -396,65 +398,68 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SArray *rsmaDeleted = NULL; - // step 1: merge rsmaInfoHash and iRsmaInfoHash + // step 1: merge qTaskInfo and iQTaskInfo // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); -#if 0 - if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) { - // just switch the hash pointer if rsmaInfoHash is empty - if (taosHashGetSize(RSMA_IMU_INFO_HASH(pRSmaStat)) > 0) { - SHashObj *infoHash = RSMA_INFO_HASH(pRSmaStat); - RSMA_INFO_HASH(pRSmaStat) = RSMA_IMU_INFO_HASH(pRSmaStat); - RSMA_IMU_INFO_HASH(pRSmaStat) = infoHash; - } - } else { -#endif -#if 1 - void *pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), NULL); + + void *pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); while (pIter) { tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - - if (!taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t))) { - SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; - if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - int32_t refVal = T_REF_VAL_GET(pRSmaInfo); - if (refVal == 0) { - tdFreeRSmaInfo(pSma, pRSmaInfo, true); - smaDebug( - "vgId:%d, rsma async post commit, free rsma info since already deleted and ref is 0 for " - "table:%" PRIi64, - SMA_VID(pSma), *pSuid); - } else { - smaDebug( - "vgId:%d, rsma async post commit, not free rsma info since ref is %d although already deleted for " - "table:%" PRIi64, - SMA_VID(pSma), refVal, *pSuid); + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + int32_t refVal = T_REF_VAL_GET(pRSmaInfo); + if (refVal == 0) { + if(!rsmaDeleted) { + if((rsmaDeleted = taosArrayInit(1, sizeof(tb_uid_t)))){ + taosArrayPush(rsmaDeleted, pSuid); + } } - - pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter); - continue; + } else { + smaDebug( + "vgId:%d, rsma async post commit, not free rsma info since ref is %d although already deleted for " + "table:%" PRIi64, + SMA_VID(pSma), refVal, *pSuid); } - taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); - smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), - *pSuid); - } else { - // free the resources - SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; - tdFreeRSmaInfo(pSma, pRSmaInfo, false); - smaDebug("vgId:%d, rsma async post commit, free rsma info since already COW for table:%" PRIi64, SMA_VID(pSma), - *pSuid); + + pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter); + continue; } - pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter); - } -#endif - // } + if (pRSmaInfo->taskInfo[0]) { + if (pRSmaInfo->iTaskInfo[0]) { + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pRSmaInfo->iTaskInfo[0]; + tdFreeRSmaInfo(pSma, pRSmaInfo, true); + pRSmaInfo->iTaskInfo[0] = NULL; + } + } else { + TSWAP(pRSmaInfo->taskInfo[0], pRSmaInfo->iTaskInfo[0]); + } - taosHashCleanup(RSMA_IMU_INFO_HASH(pRSmaStat)); - RSMA_IMU_INFO_HASH(pRSmaStat) = NULL; + taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); + smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), *pSuid); + + pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter); + } + + if (taosArrayGetSize(rsmaDeleted) > 0) { + for (int32_t i = 0; i < taosArrayGetSize(rsmaDeleted); ++i) { + tb_uid_t *pSuid = taosArrayGet(rsmaDeleted, i); + void *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t)); + if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + tdFreeRSmaInfo(pSma, pRSmaInfo, true); + smaDebug( + "vgId:%d, rsma async post commit, free rsma info since already deleted and ref is 0 for " + "table:%" PRIi64, + SMA_VID(pSma), *pSuid); + } + taosHashRemove(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t)); + } + // remove suid in files + taosArrayDestroy(rsmaDeleted); + } // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 50db1123ae..73f8060559 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -315,9 +315,9 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (pSmaStat) { if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat)); + tdDestroyTSmaStat(SMA_STAT_TSMA(pSmaStat)); } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat); + SRSmaStat *pRSmaStat = &pSmaStat->rsmaStat; int32_t vid = SMA_VID(pRSmaStat->pSma); int64_t refId = RSMA_REF_ID(pRSmaStat); if (taosRemoveRef(smaMgmt.rsetId, RSMA_REF_ID(pRSmaStat)) < 0) { diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 1660223d77..41393eb52f 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -17,6 +17,7 @@ #define RSMA_QTASKINFO_BUFSIZE 32768 #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid +#define RSMA_QTASKEXEC_BUFSIZ 1 // * 1048576 // 8 MB SSmaMgmt smaMgmt = { .inited = 0, @@ -27,17 +28,18 @@ SSmaMgmt smaMgmt = { #define TD_RSMAINFO_DEL_FILE "rsmainfo.del" typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter; +typedef struct SRSmaExecQItem SRSmaExecQItem; static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx); -static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid, - int8_t level); +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo, + tb_uid_t suid, int8_t level); static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid); static void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, - int64_t suid, int8_t blkType); + int64_t suid); static void tdRSmaFetchTrigger(void *param, void *tmrId); static int32_t tdRSmaFetchSend(SSma *pSma, SRSmaInfo *pInfo, int8_t level); static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); @@ -76,6 +78,11 @@ struct SRSmaQTaskInfoIter { int32_t nBufPos; }; +struct SRSmaExecQItem { + void *pRSmaInfo; + void *qall; +}; + void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t version, char *outputName) { tdGetVndFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName); } @@ -143,8 +150,12 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { if (isDeepFree) { if (pInfo->queue) taosCloseQueue(pInfo->queue); if (pInfo->qall) taosFreeQall(pInfo->qall); + if (pInfo->iQueue) taosCloseQueue(pInfo->iQueue); + if (pInfo->iQall) taosFreeQall(pInfo->iQall); pInfo->queue = NULL; pInfo->qall = NULL; + pInfo->iQueue = NULL; + pInfo->iQall = NULL; } taosMemoryFree(pInfo); @@ -362,9 +373,18 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con if (!(pRSmaInfo->queue = taosOpenQueue())) { goto _err; } + smaError("vgId:%d init bufSize:%" PRIi64 ", qMemSize:%" PRIi64, SMA_VID(pSma), atomic_load_64(&pStat->qBufSize), + taosQueueMemorySize(pRSmaInfo->queue)); + if (!(pRSmaInfo->qall = taosAllocateQall())) { goto _err; } + if (!(pRSmaInfo->iQueue = taosOpenQueue())) { + goto _err; + } + if (!(pRSmaInfo->iQall = taosAllocateQall())) { + goto _err; + } pRSmaInfo->suid = suid; pRSmaInfo->refId = RSMA_REF_ID(pStat); T_REF_INIT_VAL(pRSmaInfo, 1); @@ -433,8 +453,7 @@ int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pReq->suid); @@ -619,7 +638,7 @@ _end: } static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, - int64_t suid, int8_t blkType) { + int64_t suid) { SArray *pResList = taosArrayInit(1, POINTER_BYTES); if (pResList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -690,8 +709,50 @@ _err: return TSDB_CODE_FAILED; } -static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid, - int8_t level) { +/** + * @brief Copy msg to rsmaQueueBuffer + * + * @param pSma + * @param pMsg + * @param inputType + * @param pInfo + * @param suid + * @return int32_t + */ +static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, + tb_uid_t suid) { + const SSubmitReq *pReq = (const SSubmitReq *)pMsg; + + void *qItem = taosAllocateQitem(pReq->length, DEF_QITEM); + if (!qItem) { + return TSDB_CODE_FAILED; + } + + memcpy(qItem, pMsg, pReq->header.contLen); + + taosWriteQitem(pInfo->queue, qItem); + + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); + int64_t size = atomic_fetch_add_64(&pRSmaStat->qBufSize, taosQueueMemorySize(pInfo->queue)); + smaError("vgId:%d originSize:%" PRIi64 ", after push size is:%" PRIi64, SMA_VID(pSma), size, + atomic_load_64(&pRSmaStat->qBufSize)); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief sync mode + * + * @param pSma + * @param pMsg + * @param msgSize + * @param inputType + * @param pInfo + * @param suid + * @param level + * @return int32_t + */ +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo, + tb_uid_t suid, int8_t level) { int32_t idx = level - 1; if (!pInfo || !RSMA_INFO_QTASK(pInfo, idx)) { smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); @@ -705,14 +766,13 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, RSMA_INFO_QTASK(pInfo, idx), suid); - if (qSetMultiStreamInput(RSMA_INFO_QTASK(pInfo, idx), pMsg, 1, inputType) < 0) { // INPUT__DATA_SUBMIT + if (qSetMultiStreamInput(RSMA_INFO_QTASK(pInfo, idx), pMsg, msgSize, inputType) < 0) { // INPUT__DATA_SUBMIT smaError("vgId:%d, rsma %" PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); - tdRSmaFetchAndSubmitResult(pSma, RSMA_INFO_QTASK(pInfo, idx), pItem, pInfo->pTSchema, suid, - STREAM_INPUT__DATA_SUBMIT); + tdRSmaFetchAndSubmitResult(pSma, RSMA_INFO_QTASK(pInfo, idx), pItem, pInfo->pTSchema, suid); atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); if (smaMgmt.tmrHandle) { @@ -752,8 +812,15 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } + if (!pRSmaInfo->taskInfo[0]) { + if (tdCloneRSmaInfo(pSma, pRSmaInfo) < 0) { + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + return NULL; + } + } tdRefRSmaInfo(pSma, pRSmaInfo); taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + ASSERT(pRSmaInfo->suid == suid); return pRSmaInfo; } taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); @@ -762,41 +829,9 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { return NULL; } - // clone the SRSmaInfo from iRsmaInfoHash to rsmaInfoHash if in committing stat - SRSmaInfo *pCowRSmaInfo = NULL; - // lock - taosWLockLatch(SMA_ENV_LOCK(pEnv)); - if (!(pCowRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)))) { // 2-phase lock - void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); - if (iRSmaInfo) { - SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; - if (pIRSmaInfo && !RSMA_INFO_IS_DEL(pIRSmaInfo)) { - if (tdCloneRSmaInfo(pSma, &pCowRSmaInfo, pIRSmaInfo) < 0) { - // unlock - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); - return NULL; - } - smaDebug("vgId:%d, clone rsma info succeed for suid:%" PRIu64, SMA_VID(pSma), suid); - if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) { - // unlock - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); - return NULL; - } - } - } - } else { - pCowRSmaInfo = *(SRSmaInfo **)pCowRSmaInfo; - ASSERT(!pCowRSmaInfo); - } - - if (pCowRSmaInfo) { - tdRefRSmaInfo(pSma, pCowRSmaInfo); - } // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - return pCowRSmaInfo; + return pRSmaInfo; } static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { @@ -805,10 +840,47 @@ static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { } } +/** + * @brief async mode + * + * @param pSma + * @param pMsg + * @param inputType + * @param suid + * @return int32_t + */ +static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { + SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid); + if (!pRSmaInfo) { + smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); + return TSDB_CODE_SUCCESS; + } + + if (inputType == STREAM_INPUT__DATA_SUBMIT) { + if (tdExecuteRSmaImplAsync(pSma, pMsg, inputType, pRSmaInfo, suid) < 0) { + tdReleaseRSmaInfo(pSma, pRSmaInfo); + return TSDB_CODE_FAILED; + } + } + + tdReleaseRSmaInfo(pSma, pRSmaInfo); + return TSDB_CODE_SUCCESS; +} + +#if 0 +/** + * @brief sync mode + * + * @param pSma + * @param pMsg + * @param inputType + * @param suid + * @return int32_t + */ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid); if (!pRSmaInfo) { - smaError("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); + smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } @@ -820,6 +892,47 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_SUCCESS; } +#endif + +static int32_t tdRSmaExecCheck(SSma *pSma) { + SRSmaStat *pRsmaStat = SMA_RSMA_STAT(pSma); + int64_t bufSize = atomic_load_64(&pRsmaStat->qBufSize); + + if ((pRsmaStat->execStat == 1) || (bufSize < RSMA_QTASKEXEC_BUFSIZ)) { + smaError("vgId:%d, return directly as execStat:%" PRIi8 ", bufSize:%" PRIi64, SMA_VID(pSma), pRsmaStat->execStat, + bufSize); + return TSDB_CODE_SUCCESS; + } + smaError("vgId:%d, go on exec as execStat:%" PRIi8 ", bufSize:%" PRIi64, SMA_VID(pSma), pRsmaStat->execStat, bufSize); + + pRsmaStat->execStat = 1; + + SRSmaExecMsg fetchMsg; + int32_t contLen = sizeof(SMsgHead); + void *pBuf = rpcMallocCont(0 + contLen); + + ((SMsgHead *)pBuf)->vgId = SMA_VID(pSma); + ((SMsgHead *)pBuf)->contLen = sizeof(SMsgHead); + + SRpcMsg rpcMsg = { + .code = 0, + .msgType = TDMT_VND_EXEC_RSMA, + .pCont = pBuf, + .contLen = contLen, + }; + + if ((terrno = tmsgPutToQueue(&pSma->pVnode->msgCb, QUERY_QUEUE, &rpcMsg)) != 0) { + smaError("vgId:%d, failed to put rsma exec msg into query-queue since %s", SMA_VID(pSma), terrstr()); + goto _err; + } + + smaDebug("vgId:%d, success to put rsma fetch msg into query-queue", SMA_VID(pSma)); + + return TSDB_CODE_SUCCESS; +_err: + pRsmaStat->execStat = 0; + return TSDB_CODE_FAILED; +} int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); @@ -839,16 +952,18 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { tdFetchSubmitReqSuids(pMsg, &uidStore); if (uidStore.suid != 0) { - tdExecuteRSma(pSma, pMsg, inputType, uidStore.suid); + tdExecuteRSmaAsync(pSma, pMsg, inputType, uidStore.suid); void *pIter = taosHashIterate(uidStore.uidHash, NULL); while (pIter) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - tdExecuteRSma(pSma, pMsg, inputType, *pTbSuid); + tdExecuteRSmaAsync(pSma, pMsg, inputType, *pTbSuid); pIter = taosHashIterate(uidStore.uidHash, pIter); } tdUidStoreDestory(&uidStore); + + tdRSmaExecCheck(pSma); } } return TSDB_CODE_SUCCESS; @@ -1282,7 +1397,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { } for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pRSmaInfo, i); + qTaskInfo_t taskInfo = RSMA_INFO_IQTASK(pRSmaInfo, i); if (!taskInfo) { smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); continue; @@ -1452,7 +1567,7 @@ _end: * @param level * @return int32_t */ -int32_t tdRSmaFetchSend(SSma *pSma, SRSmaInfo *pInfo, int8_t level) { +static int32_t tdRSmaFetchSend(SSma *pSma, SRSmaInfo *pInfo, int8_t level) { SRSmaFetchMsg fetchMsg = {.suid = pInfo->suid, .level = level}; int32_t ret = 0; int32_t contLen = 0; @@ -1479,7 +1594,7 @@ int32_t tdRSmaFetchSend(SSma *pSma, SRSmaInfo *pInfo, int8_t level) { .code = 0, .msgType = TDMT_VND_FETCH_RSMA, .pCont = pBuf, - .contLen = contLen, + .contLen = contLen + sizeof(SMsgHead), }; if ((terrno = tmsgPutToQueue(&pSma->pVnode->msgCb, QUERY_QUEUE, &rpcMsg)) != 0) { @@ -1541,7 +1656,7 @@ int32_t smaProcessFetch(SSma *pSma, void *pMsg) { if ((terrno = qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) { goto _err; } - if (tdRSmaFetchAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid, STREAM_INPUT__DATA_BLOCK) < 0) { + if (tdRSmaFetchAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid) < 0) { goto _err; } @@ -1558,3 +1673,125 @@ _err: smaError("vgId:%d, failed to process rsma fetch msg since %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; } + +static void tdFreeRSmaSubmitItems(SArray *pItems) { + for (int32_t i = 0; i < taosArrayGetSize(pItems); ++i) { + taosFreeQitem(*(void **)taosArrayGet(pItems, i)); + } +} + +int32_t tdRSmaProcessExecImpl(SSma *pSma) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SHashObj *infoHash = NULL; + SArray *pSubmitQArr = NULL; + SArray *pSubmitArr = NULL; + + if (!pRSmaStat || !(infoHash = RSMA_INFO_HASH(pRSmaStat))) { + terrno = TSDB_CODE_RSMA_INVALID_STAT; + goto _err; + } + + taosRLockLatch(SMA_ENV_LOCK(pEnv)); + if (atomic_load_64(&pRSmaStat->qBufSize) < RSMA_QTASKEXEC_BUFSIZ) { + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + return TSDB_CODE_SUCCESS; + } + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + + if (!(pSubmitQArr = taosArrayInit(taosHashGetSize(infoHash), sizeof(SRSmaExecQItem)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + if (!(pSubmitArr = taosArrayInit(1024, POINTER_BYTES))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + SRSmaExecQItem qItem = {0}; + taosWLockLatch(SMA_ENV_LOCK(pEnv)); + void *pIter = taosHashIterate(infoHash, NULL); + while (pIter) { + SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; + if (taosQueueItemSize(pInfo->queue)) { + taosReadAllQitems(pInfo->queue, pInfo->qall); + qItem.qall = &pInfo->qall; + qItem.pRSmaInfo = pIter; + taosArrayPush(pSubmitQArr, &qItem); + } + ASSERT(taosQueueItemSize(pInfo->queue) == 0); + pIter = taosHashIterate(infoHash, pIter); + } + + atomic_store_64(&pRSmaStat->qBufSize, 0); + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + smaError("vgId:%d after exec qBufSize is:%" PRIi64, SMA_VID(pSma), atomic_load_64(&pRSmaStat->qBufSize)); + + int32_t qSize = taosArrayGetSize(pSubmitQArr); + for (int32_t i = 0; i < qSize; ++i) { + SRSmaExecQItem *pItem = taosArrayGet(pSubmitQArr, i); + while (1) { + void *msg = NULL; + taosGetQitem(*(STaosQall **)pItem->qall, (void **)&msg); + if (msg) { + if (taosArrayPush(pSubmitArr, &msg) < 0) { + tdFreeRSmaSubmitItems(pSubmitArr); + goto _err; + } + } else { + break; + } + } + + int32_t size = taosArrayGetSize(pSubmitArr); + if (size > 0) { + SRSmaInfo *pInfo = *(SRSmaInfo **)pItem->pRSmaInfo; + for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { + if (tdExecuteRSmaImpl(pSma, *(SSubmitReq**)pSubmitArr->pData, size, STREAM_INPUT__DATA_SUBMIT, pInfo, pInfo->suid, i) < 0) { + tdFreeRSmaSubmitItems(pSubmitArr); + goto _err; + } + } + tdFreeRSmaSubmitItems(pSubmitArr); + taosArrayClear(pSubmitArr); + } + } + + taosArrayDestroy(pSubmitArr); + taosArrayDestroy(pSubmitQArr); + return TSDB_CODE_SUCCESS; +_err: + taosArrayDestroy(pSubmitArr); + taosArrayDestroy(pSubmitQArr); + return TSDB_CODE_FAILED; +} + +/** + * @brief exec rsma level 1data, fetch result of level 2/3 and submit + * + * @param pSma + * @param pMsg + * @return int32_t + */ +int32_t smaProcessExec(SSma *pSma, void *pMsg) { + SRpcMsg *pRpcMsg = (SRpcMsg *)pMsg; + SRSmaStat *pRsmaStat = SMA_RSMA_STAT(pSma); + + if (!pRpcMsg || pRpcMsg->contLen < sizeof(SMsgHead)) { + terrno = TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP; + goto _err; + } + + if (tdRSmaProcessExecImpl(pSma) < 0) { + goto _err; + } + + pRsmaStat->execStat = 0; + smaWarn("vgId:%d, success to process rsma exec msg", SMA_VID(pSma)); + return TSDB_CODE_SUCCESS; +_err: + pRsmaStat->execStat = 0; + smaError("vgId:%d, failed to process rsma fetch msg since %s", SMA_VID(pSma), terrstr()); + return TSDB_CODE_FAILED; +} diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index f46d9dc29c..a6302b9235 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -175,7 +175,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { } tdRefSmaStat(pSma, pStat); - pTsmaStat = SMA_TSMA_STAT(pStat); + pTsmaStat = SMA_STAT_TSMA(pStat); if (!pTsmaStat->pTSma) { STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index d9f38ffd09..da70222485 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -350,49 +350,45 @@ _err: } /** - * @brief pTSchema is shared + * @brief Clone qTaskInfo of SRSmaInfo * * @param pSma - * @param pDest - * @param pSrc + * @param pInfo * @return int32_t */ -int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo **pDest, SRSmaInfo *pSrc) { - SVnode *pVnode = pSma->pVnode; +int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { SRSmaParam *param = NULL; - if (!pSrc) { - *pDest = NULL; + if (!pInfo) { return TSDB_CODE_SUCCESS; } SMetaReader mr = {0}; metaReaderInit(&mr, SMA_META(pSma), 0); - smaDebug("vgId:%d, rsma clone, suid is %" PRIi64, TD_VID(pVnode), pSrc->suid); - if (metaGetTableEntryByUid(&mr, pSrc->suid) < 0) { - smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), pSrc->suid, + smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid); + if (metaGetTableEntryByUid(&mr, pInfo->suid) < 0) { + smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, terrstr()); goto _err; } ASSERT(mr.me.type == TSDB_SUPER_TABLE); - ASSERT(mr.me.uid == pSrc->suid); + ASSERT(mr.me.uid == pInfo->suid); if (TABLE_IS_ROLLUP(mr.me.flags)) { param = &mr.me.stbEntry.rsmaParam; for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - if (tdCloneQTaskInfo(pSma, pSrc->iTaskInfo[i], pSrc->taskInfo[i], param, pSrc->suid, i) < 0) { + if (tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i) < 0) { goto _err; } } - smaDebug("vgId:%d, rsma clone env success for %" PRIi64, TD_VID(pVnode), pSrc->suid); + smaDebug("vgId:%d, rsma clone env success for %" PRIi64, SMA_VID(pSma), pInfo->suid); + } else { + terrno = TSDB_CODE_RSMA_INVALID_SCHEMA; + goto _err; } metaReaderClear(&mr); - - *pDest = pSrc; // pointer copy - return TSDB_CODE_SUCCESS; _err: - *pDest = NULL; metaReaderClear(&mr); - smaError("vgId:%d, rsma clone env failed for %" PRIi64 " since %s", TD_VID(pVnode), pSrc->suid, terrstr()); + smaError("vgId:%d, rsma clone env failed for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, terrstr()); return TSDB_CODE_FAILED; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 3a25933ec4..751cb21d08 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -295,6 +295,8 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg, 0); case TDMT_VND_FETCH_RSMA: return smaProcessFetch(pVnode->pSma, pMsg); + case TDMT_VND_EXEC_RSMA: + return smaProcessExec(pVnode->pSma, pMsg); default: vError("unknown msg type:%d in query queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 7115ad85a5..4c3d5cf7af 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -55,7 +55,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayClear(pInfo->pBlockLists); if (type == STREAM_INPUT__MERGED_SUBMIT) { - ASSERT(numOfBlocks > 1); + // ASSERT(numOfBlocks > 1); for (int32_t i = 0; i < numOfBlocks; i++) { SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*)); taosArrayPush(pInfo->pBlockLists, &pReq); diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index dbb50f958d..6b2edf0d5e 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -15,6 +15,7 @@ #include "tsimplehash.h" #include "taoserror.h" +#include "tlog.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 #define HASH_MAX_CAPACITY (1024 * 1024 * 16) @@ -106,27 +107,27 @@ static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *data return pNewNode; } -static void taosHashTableResize(SSHashObj *pHashObj) { +static void tSimpleHashTableResize(SSHashObj *pHashObj) { if (!SHASH_NEED_RESIZE(pHashObj)) { return; } int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); if (newCapacity > HASH_MAX_CAPACITY) { - // uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", - // pHashObj->capacity, HASH_MAX_CAPACITY); + uDebug("current capacity:%zu, maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached", + pHashObj->capacity, HASH_MAX_CAPACITY); return; } int64_t st = taosGetTimestampUs(); void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (!pNewEntryList) { - // qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); + uWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; } size_t inc = newCapacity - pHashObj->capacity; - memset((char *)pNewEntryList + pHashObj->capacity * sizeof(void *), 0, inc); + memset((char *)pNewEntryList + pHashObj->capacity * sizeof(void *), 0, inc * sizeof(void *)); pHashObj->hashList = pNewEntryList; pHashObj->capacity = newCapacity; @@ -179,7 +180,7 @@ int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, size_t keyLen, cons // need the resize process, write lock applied if (SHASH_NEED_RESIZE(pHashObj)) { - taosHashTableResize(pHashObj); + tSimpleHashTableResize(pHashObj); } int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6e3067d44e..3f6d3421ec 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -616,6 +616,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FILE_CORRUPTED, "Rsma file corrupted TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REMOVE_EXISTS, "Rsma remove exists") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP, "Rsma fetch msg is messed up") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_EMPTY_INFO, "Rsma info is empty") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_SCHEMA, "Rsma invalid schema") //index TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding") From 34e14f770fd4080058ac07b2936d33ea7a2bd5f2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 15 Aug 2022 09:39:55 +0800 Subject: [PATCH 040/194] enh: rsma batch process --- source/dnode/vnode/src/sma/smaRollup.c | 17 ++---- source/libs/executor/inc/executil.h | 5 +- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/executil.c | 11 ++-- source/libs/executor/src/executorimpl.c | 24 ++++---- source/libs/executor/src/scanoperator.c | 6 +- source/libs/executor/src/timewindowoperator.c | 56 +++++++++---------- 7 files changed, 54 insertions(+), 69 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 41393eb52f..4ffd6479f5 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -17,7 +17,7 @@ #define RSMA_QTASKINFO_BUFSIZE 32768 #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid -#define RSMA_QTASKEXEC_BUFSIZ 1 // * 1048576 // 8 MB +#define RSMA_QTASKEXEC_BUFSIZ 10 * 1048576 // 8 MB SSmaMgmt smaMgmt = { .inited = 0, @@ -373,8 +373,6 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con if (!(pRSmaInfo->queue = taosOpenQueue())) { goto _err; } - smaError("vgId:%d init bufSize:%" PRIi64 ", qMemSize:%" PRIi64, SMA_VID(pSma), atomic_load_64(&pStat->qBufSize), - taosQueueMemorySize(pRSmaInfo->queue)); if (!(pRSmaInfo->qall = taosAllocateQall())) { goto _err; @@ -723,7 +721,7 @@ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inpu tb_uid_t suid) { const SSubmitReq *pReq = (const SSubmitReq *)pMsg; - void *qItem = taosAllocateQitem(pReq->length, DEF_QITEM); + void *qItem = taosAllocateQitem(pReq->header.contLen, DEF_QITEM); if (!qItem) { return TSDB_CODE_FAILED; } @@ -733,9 +731,8 @@ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inpu taosWriteQitem(pInfo->queue, qItem); SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); - int64_t size = atomic_fetch_add_64(&pRSmaStat->qBufSize, taosQueueMemorySize(pInfo->queue)); - smaError("vgId:%d originSize:%" PRIi64 ", after push size is:%" PRIi64, SMA_VID(pSma), size, - atomic_load_64(&pRSmaStat->qBufSize)); + atomic_fetch_add_64(&pRSmaStat->qBufSize, taosQueueMemorySize(pInfo->queue)); + return TSDB_CODE_SUCCESS; } @@ -899,11 +896,8 @@ static int32_t tdRSmaExecCheck(SSma *pSma) { int64_t bufSize = atomic_load_64(&pRsmaStat->qBufSize); if ((pRsmaStat->execStat == 1) || (bufSize < RSMA_QTASKEXEC_BUFSIZ)) { - smaError("vgId:%d, return directly as execStat:%" PRIi8 ", bufSize:%" PRIi64, SMA_VID(pSma), pRsmaStat->execStat, - bufSize); return TSDB_CODE_SUCCESS; } - smaError("vgId:%d, go on exec as execStat:%" PRIi8 ", bufSize:%" PRIi64, SMA_VID(pSma), pRsmaStat->execStat, bufSize); pRsmaStat->execStat = 1; @@ -1726,7 +1720,6 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma) { atomic_store_64(&pRSmaStat->qBufSize, 0); taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - smaError("vgId:%d after exec qBufSize is:%" PRIi64, SMA_VID(pSma), atomic_load_64(&pRSmaStat->qBufSize)); int32_t qSize = taosArrayGetSize(pSubmitQArr); for (int32_t i = 0; i < qSize; ++i) { @@ -1748,7 +1741,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma) { if (size > 0) { SRSmaInfo *pInfo = *(SRSmaInfo **)pItem->pRSmaInfo; for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { - if (tdExecuteRSmaImpl(pSma, *(SSubmitReq**)pSubmitArr->pData, size, STREAM_INPUT__DATA_SUBMIT, pInfo, pInfo->suid, i) < 0) { + if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, pInfo->suid, i) < 0) { tdFreeRSmaSubmitItems(pSubmitArr); goto _err; } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 58b2c1b095..d5b979c762 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -22,7 +22,6 @@ #include "tbuffer.h" #include "tcommon.h" #include "tpagedbuf.h" -#include "tsimplehash.h" #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ do { \ @@ -103,7 +102,7 @@ static FORCE_INLINE void setResultBufPageDirty(SDiskbasedBuf* pBuf, SResultRowPo setBufPageDirty(pPage, true); } -void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t order); +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); @@ -141,4 +140,4 @@ int32_t resultrowComparAsc(const void* p1, const void* p2); int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); -#endif // TDENGINE_QUERYUTIL_H +#endif // TDENGINE_QUERYUTIL_H \ No newline at end of file diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 585b642d2b..11d371d49b 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -296,7 +296,7 @@ enum { }; typedef struct SAggSupporter { - SSHashObj* pResultRowHashTable; // quick locate the window object for each result + SHashObj* pResultRowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row @@ -1045,4 +1045,4 @@ void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); } #endif -#endif // TDENGINE_EXECUTORIMPL_H +#endif // TDENGINE_EXECUTORIMPL_H \ No newline at end of file diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2e6bd312f3..615d742d40 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -97,7 +97,7 @@ int32_t resultrowComparAsc(const void* p1, const void* p2) { static int32_t resultrowComparDesc(const void* p1, const void* p2) { return resultrowComparAsc(p2, p1); } -void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t order) { +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order) { if (pGroupResInfo->pRows != NULL) { taosArrayDestroy(pGroupResInfo->pRows); } @@ -106,10 +106,9 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in void* pData = NULL; pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES); - size_t keyLen = 0; - int32_t iter = 0; - while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { - void* key = tSimpleHashGetKey(pData, &keyLen); + size_t keyLen = 0; + while ((pData = taosHashIterate(pHashmap, pData)) != NULL) { + void* key = taosHashGetKey(pData, &keyLen); SResKeyPos* p = taosMemoryMalloc(keyLen + sizeof(SResultRowPosition)); @@ -987,4 +986,4 @@ void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimit pLimitInfo->slimit = slimit; pLimitInfo->remainOffset = limit.offset; pLimitInfo->remainGroupOffset = slimit.offset; -} +} \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 46ca99c8cd..5d07a15b2f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -250,7 +250,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); SResultRow* pResult = NULL; @@ -292,7 +292,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // add a new result set for a new group SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; - tSimpleHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, + taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, sizeof(SResultRowPosition)); } @@ -301,7 +301,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // too many time window in query if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH && - tSimpleHashGetSize(pSup->pResultRowHashTable) > MAX_INTERVAL_TIME_WINDOW) { + taosHashGetSize(pSup->pResultRowHashTable) > MAX_INTERVAL_TIME_WINDOW) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } @@ -3017,7 +3017,7 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len } SOptrBasicInfo* pInfo = (SOptrBasicInfo*)(pOperator->info); SAggSupporter* pSup = (SAggSupporter*)POINTER_SHIFT(pOperator->info, sizeof(SOptrBasicInfo)); - int32_t size = tSimpleHashGetSize(pSup->pResultRowHashTable); + int32_t size = taosHashGetSize(pSup->pResultRowHashTable); size_t keyLen = sizeof(uint64_t) * 2; // estimate the key length int32_t totalSize = sizeof(int32_t) + sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); @@ -3045,10 +3045,9 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len setBufPageDirty(pPage, true); releaseBufPage(pSup->pResultBuf, pPage); - int32_t iter = 0; - void* pIter = tSimpleHashIterate(pSup->pResultRowHashTable, NULL, &iter); + void* pIter = taosHashIterate(pSup->pResultRowHashTable, NULL); while (pIter) { - void* key = tSimpleHashGetKey(pIter, &keyLen); + void* key = taosHashGetKey(pIter, &keyLen); SResultRowPosition* p1 = (SResultRowPosition*)pIter; pPage = (SFilePage*)getBufPage(pSup->pResultBuf, p1->pageId); @@ -3080,7 +3079,7 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len memcpy(*result + offset, pRow, pSup->resultRowSize); offset += pSup->resultRowSize; - pIter = tSimpleHashIterate(pSup->pResultRowHashTable, pIter, &iter); + pIter = taosHashIterate(pSup->pResultRowHashTable, pIter); } *(int32_t*)(*result) = offset; @@ -3115,7 +3114,7 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) { // add a new result set for a new group SResultRowPosition pos = {.pageId = resultRow->pageId, .offset = resultRow->offset}; - tSimpleHashPut(pSup->pResultRowHashTable, result + offset, keyLen, &pos, sizeof(SResultRowPosition)); + taosHashPut(pSup->pResultRowHashTable, result + offset, keyLen, &pos, sizeof(SResultRowPosition)); offset += keyLen; int32_t valueLen = *(int32_t*)(result + offset); @@ -3454,8 +3453,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); - // pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); - pAggSup->pResultRowHashTable = tSimpleHashInit(100000, hashFn); + pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3481,7 +3479,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n void cleanupAggSup(SAggSupporter* pAggSup) { taosMemoryFreeClear(pAggSup->keyBuf); - tSimpleHashCleanup(pAggSup->pResultRowHashTable); + taosHashCleanup(pAggSup->pResultRowHashTable); destroyDiskbasedBuf(pAggSup->pResultBuf); } @@ -4779,4 +4777,4 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF pCtx[i].pBuf = pSup->pResultBuf; } return code; -} +} \ No newline at end of file diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c404fca597..c373634c16 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -178,8 +178,8 @@ static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t gro STableScanInfo* pTableScanInfo = pOperator->info; - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pTableScanInfo->pdInfo.pAggSup->pResultRowHashTable, buf, - GET_RES_WINDOW_KEY_LEN(sizeof(groupId))); + SResultRowPosition* p1 = + (SResultRowPosition*)taosHashGet(pTableScanInfo->pdInfo.pAggSup->pResultRowHashTable, buf, GET_RES_WINDOW_KEY_LEN(sizeof(groupId))); if (p1 == NULL) { return NULL; @@ -3128,4 +3128,4 @@ _error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); return NULL; -} +} \ No newline at end of file diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index ff323bf4ba..b81cb7724f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1385,7 +1385,7 @@ bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t int32_t numOfOutput) { SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (!p1) { // window has been closed return false; @@ -1398,14 +1398,14 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) size_t bytes = sizeof(TSKEY); SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (!p1) { // window has been closed return false; } // SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId); // dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage); - tSimpleHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + taosHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); return true; } @@ -1455,12 +1455,11 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* } } -static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { +static int32_t getAllIntervalWindow(SHashObj* pHashMap, SHashObj* resWins) { void* pIte = NULL; size_t keyLen = 0; - int32_t iter = 0; - while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { - void* key = tSimpleHashGetKey(pIte, &keyLen); + while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { + void* key = taosHashGetKey(pIte, &keyLen); uint64_t groupId = *(uint64_t*)key; ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); @@ -1473,18 +1472,16 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { return TSDB_CODE_SUCCESS; } -static int32_t closeIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, +static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pRecyPages, SDiskbasedBuf* pDiscBuf) { qDebug("===stream===close interval window"); void* pIte = NULL; - void* key = NULL; - size_t keyLen = GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)); - int32_t iter = 0; - while ((pIte = tSimpleHashIterateKV(pHashMap, pIte, &key, &iter)) != NULL) { - // void* key = tSimpleHashGetKey(pIte, &keyLen); + size_t keyLen = 0; + while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { + void* key = taosHashGetKey(pIte, &keyLen); uint64_t groupId = *(uint64_t*)key; - // ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); + ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); STimeWindow win; win.skey = ts; @@ -1520,7 +1517,7 @@ static int32_t closeIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pSup } char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); - tSimpleHashRemove(pHashMap, keyBuf, keyLen); + taosHashRemove(pHashMap, keyBuf, keyLen); } } return TSDB_CODE_SUCCESS; @@ -2850,7 +2847,7 @@ bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) { int32_t bytes = sizeof(TSKEY); SET_RES_WINDOW_KEY(pSup->keyBuf, &ts, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); return p1 != NULL; } @@ -2891,9 +2888,8 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId); - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); - + SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, + GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); return p1 == NULL; } @@ -3021,7 +3017,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc } static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) { - tSimpleHashClear(pInfo->aggSup.pResultRowHashTable); + taosHashClear(pInfo->aggSup.pResultRowHashTable); clearDiskbasedBuf(pInfo->aggSup.pResultBuf); cleanupResultRowInfo(&pInfo->binfo.resultRowInfo); initResultRowInfo(&pInfo->binfo.resultRowInfo); @@ -4932,14 +4928,14 @@ static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, ui SExprSupp* pSup = &pOperatorInfo->exprSupp; SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId); - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet( - iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, + GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); ASSERT(p1 != NULL); finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pSup->pCtx, pSup->pExprInfo, pSup->numOfExprs, pSup->rowEntryInfoOffset, pResultBlock, pTaskInfo); - tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); - ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); + taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); return TSDB_CODE_SUCCESS; } @@ -4962,7 +4958,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR // there is an result exists if (miaInfo->curTs != INT64_MIN) { - ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); + ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); if (ts != miaInfo->curTs) { outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs); @@ -4970,7 +4966,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR } } else { miaInfo->curTs = ts; - ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); + ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); } STimeWindow win = {0}; @@ -5046,7 +5042,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { if (pBlock == NULL) { // close last unfinalized time window if (miaInfo->curTs != INT64_MIN) { - ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); + ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); miaInfo->curTs = INT64_MIN; } @@ -5223,12 +5219,12 @@ static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t table SExprSupp* pExprSup = &pOperatorInfo->exprSupp; SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId); - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, + SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); ASSERT(p1 != NULL); finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo); - tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); return TSDB_CODE_SUCCESS; } @@ -5491,4 +5487,4 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; -} +} \ No newline at end of file From 5938ec6a7700bde07799497f4d18fa588c6bfaa5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Aug 2022 08:44:41 +0000 Subject: [PATCH 041/194] more work --- source/util/src/trbtree.c | 210 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 source/util/src/trbtree.c diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c new file mode 100644 index 0000000000..0970485dad --- /dev/null +++ b/source/util/src/trbtree.c @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +typedef int32_t (*tRBTreeCmprFn)(void *, void *); + +typedef struct SRBTree SRBTree; +typedef struct SRBTreeNode SRBTreeNode; +typedef struct SRBTreeIter SRBTreeIter; + +struct SRBTreeNode { + enum { RED, BLACK } color; + SRBTreeNode *parent; + SRBTreeNode *left; + SRBTreeNode *right; + uint8_t payload[]; +}; + +struct SRBTree { + tRBTreeCmprFn cmprFn; + SRBTreeNode *root; +}; + +struct SRBTreeIter { + SRBTree *pTree; +}; + +#define RBTREE_NODE_COLOR(N) ((N) ? (N)->color : BLACK) + +// APIs ================================================ +static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *pNode) { + SRBTreeNode *right = pNode->right; + + pNode->right = right->left; + if (pNode->right) { + pNode->right->parent = pNode; + } + + right->parent = pNode->parent; + if (pNode->parent == NULL) { + pTree->root = right; + } else if (pNode == pNode->parent->left) { + pNode->parent->left = right; + } else { + pNode->parent->right = right; + } + + right->left = pNode; + pNode->parent = right; +} + +static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *pNode) { + SRBTreeNode *left = pNode->left; + + pNode->left = left->right; + if (pNode->left) { + pNode->left->parent = pNode; + } + + left->parent = pNode->parent; + if (pNode->parent == NULL) { + pTree->root = left; + } else if (pNode == pNode->parent->left) { + pNode->parent->left = left; + } else { + pNode->parent->right = left; + } + + left->right = pNode; + pNode->parent = left; +} + +#define tRBTreeCreate(compare) \ + (SRBTree) { .cmprFn = (compare), .root = NULL } + +SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { + pNew->left = NULL; + pNew->right = NULL; + pNew->color = RED; + + // insert + if (pTree->root == NULL) { + pNew->parent = NULL; + pTree->root = pNew; + } else { + SRBTreeNode *pNode = pTree->root; + while (true) { + ASSERT(pNode); + + int32_t c = pTree->cmprFn(pNew->payload, pNode->payload); + if (c < 0) { + if (pNode->left) { + pNode = pNode->left; + } else { + pNew->parent = pNode; + pNode->left = pNew; + break; + } + } else if (c > 0) { + if (pNode->right) { + pNode = pNode->right; + } else { + pNew->parent = pNode; + pNode->right = pNew; + break; + } + } else { + return NULL; + } + } + } + + // fix + SRBTreeNode *pNode = pNew; + while (pNode->parent && pNode->parent->color == RED) { + SRBTreeNode *p = pNode->parent; + SRBTreeNode *g = p->parent; + + if (p == g->left) { + SRBTreeNode *u = g->right; + + if (RBTREE_NODE_COLOR(u) == RED) { + p->color = BLACK; + u->color = BLACK; + g->color = RED; + pNode = g; + } else { + if (pNode == p->right) { + pNode = p; + tRBTreeRotateLeft(pTree, pNode); + } + pNode->parent->color = BLACK; + pNode->parent->parent->color = RED; + tRBTreeRotateRight(pTree, pNode->parent->parent); + } + } else { + SRBTreeNode *u = g->left; + + if (RBTREE_NODE_COLOR(u) == RED) { + p->color = BLACK; + u->color = BLACK; + g->color = RED; + } else { + if (pNode == p->left) { + pNode = p; + tRBTreeRotateRight(pTree, pNode); + } + pNode->parent->color = BLACK; + pNode->parent->parent->color = RED; + tRBTreeRotateLeft(pTree, pNode->parent->parent); + } + } + } + + pTree->root->color = BLACK; + return pNew; +} + +SRBTreeNode *tRBTreeDrop(SRBTree *pTree, void *pKey) { + SRBTreeNode *pNode = pTree->root; + + while (pNode) { + int32_t c = pTree->cmprFn(pKey, pNode->payload); + + if (c < 0) { + pNode = pNode->left; + } else if (c > 0) { + pNode = pNode->right; + } else { + break; + } + } + + if (pNode) { + // TODO + } + + return pNode; +} + +SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { + SRBTreeNode *pNode = pTree->root; + + while (pNode) { + int32_t c = pTree->cmprFn(pKey, pNode->payload); + + if (c < 0) { + pNode = pNode->left; + } else if (c > 0) { + pNode = pNode->right; + } else { + break; + } + } + + return pNode; +} From 3359b8e6204a02b5d3365c7a24ff44767f28df08 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Aug 2022 10:16:07 +0000 Subject: [PATCH 042/194] more optimize --- source/dnode/vnode/src/inc/tsdb.h | 18 ++- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 19 ++- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 144 +++++++++++++++------ source/dnode/vnode/src/tsdb/tsdbRead.c | 8 +- 5 files changed, 134 insertions(+), 59 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f46726e647..0a03334cfd 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -194,11 +194,12 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck); // tsdbMemTable ============================================================================================== // SMemTable -int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); -void tsdbMemTableDestroy(SMemTable *pMemTable); -void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData); -void tsdbRefMemTable(SMemTable *pMemTable); -void tsdbUnrefMemTable(SMemTable *pMemTable); +int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); +void tsdbMemTableDestroy(SMemTable *pMemTable); +STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid); +void tsdbRefMemTable(SMemTable *pMemTable); +void tsdbUnrefMemTable(SMemTable *pMemTable); +SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable); // STbDataIter int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); void *tsdbTbDataIterDestroy(STbDataIter *pIter); @@ -359,6 +360,7 @@ struct STbData { SDelData *pHead; SDelData *pTail; SMemSkipList sl; + STbData *next; }; struct SMemTable { @@ -372,7 +374,11 @@ struct SMemTable { int64_t maxVersion; int64_t nRow; int64_t nDel; - SArray *aTbData; // SArray + struct { + int32_t nTbData; + int32_t nBucket; + STbData **aBucket; + }; }; struct TSDBROW { diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3cff6bef27..24d6b2f385 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -742,12 +742,12 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs STbData *pMem = NULL; if (pIter->pReadSnap->pMem) { - tsdbGetTbDataFromMemTable(pIter->pReadSnap->pMem, suid, uid, &pMem); + pMem = tsdbGetTbDataFromMemTable(pIter->pReadSnap->pMem, suid, uid); } STbData *pIMem = NULL; if (pIter->pReadSnap->pIMem) { - tsdbGetTbDataFromMemTable(pIter->pReadSnap->pIMem, suid, uid, &pIMem); + pIMem = tsdbGetTbDataFromMemTable(pIter->pReadSnap->pIMem, suid, uid); } pIter->pTsdb = pTsdb; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index ae7c5f1a90..f59a8e53f1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -35,6 +35,7 @@ typedef struct { int32_t minRow; int32_t maxRow; int8_t cmprAlg; + SArray *aTbDataP; STsdbFS fs; // -------------- TSKEY nextKey; // reset by each table commit @@ -1212,8 +1213,8 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { if (code) goto _err; // commit file data impl - for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pMemTable->aTbData); iTbData++) { - STbData *pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData); + for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pCommitter->aTbDataP); iTbData++) { + STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); // move commit until current (suid, uid) code = tsdbMoveCommitData(pCommitter, *(TABLEID *)pTbData); @@ -1270,6 +1271,11 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; + pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); + if (pCommitter->aTbDataP == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } code = tsdbFSCopy(pTsdb, &pCommitter->fs); if (code) goto _err; @@ -1395,13 +1401,13 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { int32_t iDelIdx = 0; int32_t nDelIdx = taosArrayGetSize(pCommitter->aDelIdx); int32_t iTbData = 0; - int32_t nTbData = taosArrayGetSize(pMemTable->aTbData); + int32_t nTbData = taosArrayGetSize(pCommitter->aTbDataP); STbData *pTbData; SDelIdx *pDelIdx; ASSERT(nTbData > 0); - pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData); + pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; while (true) { if (pTbData == NULL && pDelIdx == NULL) break; @@ -1427,7 +1433,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { if (code) goto _err; iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData) : NULL; + pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; continue; _commit_disk_del: @@ -1443,7 +1449,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { if (code) goto _err; iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pMemTable->aTbData, iTbData) : NULL; + pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; iDelIdx++; pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; continue; @@ -1491,6 +1497,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { tsdbUnrefMemTable(pMemTable); tsdbFSDestroy(&pCommitter->fs); + taosArrayDestroy(pCommitter->aTbDataP); tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 8ae0e824cf..278b657b9e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -15,6 +15,7 @@ #include "tsdb.h" +#define MEM_MIN_HASH 1024 #define SL_MAX_LEVEL 5 #define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) @@ -49,8 +50,10 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable->maxVersion = VERSION_MIN; pMemTable->nRow = 0; pMemTable->nDel = 0; - pMemTable->aTbData = taosArrayInit(128, sizeof(STbData *)); - if (pMemTable->aTbData == NULL) { + pMemTable->nTbData = 0; + pMemTable->nBucket = MEM_MIN_HASH; + pMemTable->aBucket = (STbData **)taosMemoryCalloc(pMemTable->nBucket, sizeof(STbData *)); + if (pMemTable->aBucket == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pMemTable); goto _err; @@ -68,37 +71,30 @@ _err: void tsdbMemTableDestroy(SMemTable *pMemTable) { if (pMemTable) { vnodeBufPoolUnRef(pMemTable->pPool); - taosArrayDestroy(pMemTable->aTbData); + taosMemoryFree(pMemTable->aBucket); taosMemoryFree(pMemTable); } } -static int32_t tbDataPCmprFn(const void *p1, const void *p2) { - STbData *pTbData1 = *(STbData **)p1; - STbData *pTbData2 = *(STbData **)p2; +static FORCE_INLINE STbData *tsdbGetTbDataFromMemTableImpl(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid) { + STbData *pTbData = pMemTable->aBucket[TABS(uid) % pMemTable->nBucket]; - if (pTbData1->suid < pTbData2->suid) { - return -1; - } else if (pTbData1->suid > pTbData2->suid) { - return 1; + while (pTbData) { + if (pTbData->uid == uid) break; + pTbData = pTbData->next; } - if (pTbData1->uid < pTbData2->uid) { - return -1; - } else if (pTbData1->uid > pTbData2->uid) { - return 1; - } - - return 0; + return pTbData; } -void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) { - STbData *pTbData = &(STbData){.suid = suid, .uid = uid}; + +STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid) { + STbData *pTbData; taosRLockLatch(&pMemTable->latch); - void *p = taosArraySearch(pMemTable->aTbData, &pTbData, tbDataPCmprFn, TD_EQ); + pTbData = tsdbGetTbDataFromMemTableImpl(pMemTable, suid, uid); taosRUnLockLatch(&pMemTable->latch); - *ppTbData = p ? *(STbData **)p : NULL; + return pTbData; } int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, @@ -318,18 +314,44 @@ _exit: return pIter->pRow; } +static int32_t tsdbMemTableRehash(SMemTable *pMemTable) { + int32_t code = 0; + + int32_t nBucket = pMemTable->nBucket * 2; + STbData **aBucket = (STbData **)taosMemoryCalloc(nBucket, sizeof(STbData *)); + if (aBucket == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iBucket = 0; iBucket < pMemTable->nBucket; iBucket++) { + STbData *pTbData = pMemTable->aBucket[iBucket]; + + while (pTbData) { + STbData *pNext = pTbData->next; + + int32_t idx = TABS(pTbData->uid) % nBucket; + pTbData->next = aBucket[idx]; + aBucket[idx] = pTbData; + + pTbData = pNext; + } + } + + taosMemoryFree(pMemTable->aBucket); + pMemTable->nBucket = nBucket; + pMemTable->aBucket = aBucket; + +_exit: + return code; +} + static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) { - int32_t code = 0; - int32_t idx = 0; - STbData *pTbData = NULL; - STbData *pTbDataT = &(STbData){.suid = suid, .uid = uid}; + int32_t code = 0; // get - idx = taosArraySearchIdx(pMemTable->aTbData, &pTbDataT, tbDataPCmprFn, TD_GE); - if (idx >= 0) { - pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, idx); - if (tbDataPCmprFn(&pTbDataT, &pTbData) == 0) goto _exit; - } + STbData *pTbData = tsdbGetTbDataFromMemTableImpl(pMemTable, suid, uid); + if (pTbData) goto _exit; // create SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; @@ -365,21 +387,23 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid SL_NODE_FORWARD(pTbData->sl.pTail, iLevel) = NULL; } - void *p; - if (idx < 0) { - idx = taosArrayGetSize(pMemTable->aTbData); + taosWLockLatch(&pMemTable->latch); + + if (pMemTable->nTbData >= pMemTable->nBucket) { + code = tsdbMemTableRehash(pMemTable); + if (code) { + taosWUnLockLatch(&pMemTable->latch); + goto _err; + } } - taosWLockLatch(&pMemTable->latch); - p = taosArrayInsert(pMemTable->aTbData, idx, &pTbData); + int32_t idx = TABS(uid) % pMemTable->nBucket; + pTbData->next = pMemTable->aBucket[idx]; + pMemTable->aBucket[idx] = pTbData; + pMemTable->nTbData++; + taosWUnLockLatch(&pMemTable->latch); - tsdbDebug("vgId:%d, add table data %p at idx:%d", TD_VID(pMemTable->pTsdb->pVnode), pTbData, idx); - - if (p == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } _exit: *ppTbData = pTbData; return code; @@ -622,3 +646,41 @@ void tsdbUnrefMemTable(SMemTable *pMemTable) { tsdbMemTableDestroy(pMemTable); } } + +static FORCE_INLINE int32_t tbDataPCmprFn(const void *p1, const void *p2) { + STbData *pTbData1 = *(STbData **)p1; + STbData *pTbData2 = *(STbData **)p2; + + if (pTbData1->suid < pTbData2->suid) { + return -1; + } else if (pTbData1->suid > pTbData2->suid) { + return 1; + } + + if (pTbData1->uid < pTbData2->uid) { + return -1; + } else if (pTbData1->uid > pTbData2->uid) { + return 1; + } + + return 0; +} + +SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) { + SArray *aTbDataP = taosArrayInit(pMemTable->nTbData, sizeof(STbData *)); + if (aTbDataP == NULL) goto _exit; + + for (int32_t iBucket = 0; iBucket < pMemTable->nBucket; iBucket++) { + STbData *pTbData = pMemTable->aBucket[iBucket]; + + while (pTbData) { + taosArrayPush(aTbDataP, &pTbData); + pTbData = pTbData->next; + } + } + + taosArraySort(aTbDataP, tbDataPCmprFn); + +_exit: + return aTbDataP; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c6ae1c529c..3e2da64701 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1616,7 +1616,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea STbData* d = NULL; if (pReader->pReadSnap->pMem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid, &d); + d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); if (d != NULL) { code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); if (code == TSDB_CODE_SUCCESS) { @@ -1637,7 +1637,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea STbData* di = NULL; if (pReader->pReadSnap->pIMem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid, &di); + di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid); if (di != NULL) { code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter); if (code == TSDB_CODE_SUCCESS) { @@ -3103,7 +3103,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { STbData* d = NULL; if (pReader->pTsdb->mem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid, &d); + d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); if (d != NULL) { rows += tsdbGetNRowsInTbData(d); } @@ -3111,7 +3111,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { STbData* di = NULL; if (pReader->pTsdb->imem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid, &di); + di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid); if (di != NULL) { rows += tsdbGetNRowsInTbData(di); } From 43c84c977aa8b9897c56ea30df4baef3a92bbf08 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Aug 2022 01:45:37 +0000 Subject: [PATCH 043/194] adjust more --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 8 +++---- source/dnode/vnode/src/tsdb/tsdbFile.c | 4 ++-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 22 +++++++++---------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0a03334cfd..7e88a884c5 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -540,7 +540,6 @@ struct SHeadFile { int64_t commitID; int64_t size; int64_t offset; - int64_t loffset; }; struct SDataFile { @@ -555,6 +554,7 @@ struct SLastFile { int64_t commitID; int64_t size; + int64_t offset; }; struct SSmaFile { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index f59a8e53f1..589bce9aa5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -466,9 +466,9 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { if (pRSet) { wSet.diskId = pRSet->diskId; wSet.fid = pCommitter->commitFid; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0, .loffset = 0}; + fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; fData = *pRSet->pDataF; - fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0}; + fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; fSma = *pRSet->pSmaF; } else { SDiskID did = {0}; @@ -479,9 +479,9 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { wSet.diskId = did; wSet.fid = pCommitter->commitFid; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0, .loffset = 0}; + fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; fData = (SDataFile){.commitID = pCommitter->commitID, .size = 0}; - fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0}; + fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; fSma = (SSmaFile){.commitID = pCommitter->commitID, .size = 0}; } code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 4043cb7470..5db02fa285 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -21,7 +21,6 @@ int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) { n += tPutI64v(p ? p + n : p, pHeadFile->commitID); n += tPutI64v(p ? p + n : p, pHeadFile->size); n += tPutI64v(p ? p + n : p, pHeadFile->offset); - n += tPutI64v(p ? p + n : p, pHeadFile->loffset); return n; } @@ -32,7 +31,6 @@ static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) { n += tGetI64v(p + n, &pHeadFile->commitID); n += tGetI64v(p + n, &pHeadFile->size); n += tGetI64v(p + n, &pHeadFile->offset); - n += tGetI64v(p + n, &pHeadFile->loffset); return n; } @@ -60,6 +58,7 @@ int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile) { n += tPutI64v(p ? p + n : p, pLastFile->commitID); n += tPutI64v(p ? p + n : p, pLastFile->size); + n += tPutI64v(p ? p + n : p, pLastFile->offset); return n; } @@ -69,6 +68,7 @@ static int32_t tGetLastFile(uint8_t *p, SLastFile *pLastFile) { n += tGetI64v(p + n, &pLastFile->commitID); n += tGetI64v(p + n, &pLastFile->size); + n += tGetI64v(p + n, &pLastFile->offset); return n; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 69a4227486..324db5b111 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -502,7 +502,7 @@ _err: int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { int32_t code = 0; int64_t offset = pReader->pSet->pHeadF->offset; - int64_t size = pReader->pSet->pHeadF->loffset - offset; + int64_t size = pReader->pSet->pHeadF->size - offset; int64_t n; uint32_t delimiter; @@ -564,8 +564,8 @@ _err: int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { int32_t code = 0; - int64_t offset = pReader->pSet->pHeadF->loffset; - int64_t size = pReader->pSet->pHeadF->size - offset; + int64_t offset = pReader->pSet->pLastF->offset; + int64_t size = pReader->pSet->pLastF->size - offset; int64_t n; uint32_t delimiter; @@ -579,13 +579,13 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { if (code) goto _err; // seek - if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { + if (taosLSeekFile(pReader->pLastFD, offset, SEEK_SET) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } // read - n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); + n = taosReadFile(pReader->pLastFD, pReader->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1292,13 +1292,13 @@ _err: int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; + SLastFile *pLastFile = &pWriter->fLast; int64_t size; int64_t n; // check if (taosArrayGetSize(aBlockL) == 0) { - pHeadFile->loffset = pHeadFile->size; + pLastFile->offset = pLastFile->size; goto _exit; } @@ -1324,19 +1324,19 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pLastFD, pWriter->pBuf1, size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } // update - pHeadFile->loffset = pHeadFile->size; - pHeadFile->size += size; + pLastFile->offset = pLastFile->size; + pLastFile->size += size; _exit: tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), - pHeadFile->loffset, size); + pLastFile->offset, size); return code; _err: From d170adf2140328935fb9fb0b69f5f082de2569d8 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 16 Aug 2022 20:22:45 +0800 Subject: [PATCH 044/194] enh: rsma batch process --- source/dnode/vnode/src/inc/sma.h | 2 +- source/dnode/vnode/src/inc/vnd.h | 1 + source/dnode/vnode/src/sma/smaCommit.c | 37 +++-- source/dnode/vnode/src/sma/smaRollup.c | 174 +++++++++++++++------- source/dnode/vnode/src/vnd/vnodeBufPool.c | 12 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 8 +- source/libs/executor/src/scanoperator.c | 3 +- 7 files changed, 169 insertions(+), 68 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index bc204e032d..c36207e495 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -240,7 +240,7 @@ static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); -int32_t tdRSmaProcessExecImpl(SSma *pSma); +int32_t tdRSmaProcessExecImpl(SSma *pSma, int8_t type); int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdProcessRSmaRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer); diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 77b18b8c02..900d29b97e 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -65,6 +65,7 @@ struct SVBufPool { SVBufPool* next; SVnode* pVnode; volatile int32_t nRef; + TdThreadSpinlock lock; int64_t size; uint8_t* ptr; SVBufPoolNode* pTail; diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 807c033489..101fca3346 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -297,10 +297,9 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { } /** - * @brief Rsma async commit implementation + * @brief Rsma async commit implementation(only do some necessary light weighted task) * 1) set rsma stat TASK_TRIGGER_STAT_PAUSED * 2) Wait all running fetch task finish to fetch and put submitMsg into level 2/3 wQueue(blocking level 1 write) - * 3) * * @param pSma * @return int32_t @@ -334,12 +333,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } } - // step 3: consume the SubmitReq in buffer - if (tdRSmaProcessExecImpl(pSma) < 0) { - return TSDB_CODE_FAILED; - } - - // step 4: swap rsmaInfoHash and iRsmaInfoHash + // step 3: swap queue/qall and iQueue/iQal // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); @@ -379,9 +373,32 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } - // perform persist task for qTaskInfo SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); - tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); + + // step 1: consume the SubmitReq in buffer + int32_t nLoops = 0; + smaDebug("vgId:%d start to wait for rsma qtask free, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); + while (pRSmaStat->execStat == 1) { + taosMsleep(15); + if ((++nLoops & 63) == 0) { + smaWarn("vgId:%d 1s waited for rsma exec stat = 0, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); + sched_yield(); + } + } + pRSmaStat->execStat = 1; + smaDebug("vgId:%d end to wait for rsma qtask free, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); + + if (tdRSmaProcessExecImpl(pSma, 1) < 0) { + pRSmaStat->execStat = 0; + return TSDB_CODE_FAILED; + } + + // step 2: perform persist task for qTaskInfo operator + if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) { + pRSmaStat->execStat = 0; + return TSDB_CODE_FAILED; + } + pRSmaStat->execStat = 0; return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 4ffd6479f5..de4b7dd808 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -15,9 +15,9 @@ #include "sma.h" -#define RSMA_QTASKINFO_BUFSIZE 32768 +#define RSMA_QTASKINFO_BUFSIZE (32768) #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid -#define RSMA_QTASKEXEC_BUFSIZ 10 * 1048576 // 8 MB +#define RSMA_QTASKEXEC_BUFSIZ (1048576) SSmaMgmt smaMgmt = { .inited = 0, @@ -35,7 +35,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUi static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx); static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo, - tb_uid_t suid, int8_t level); + int8_t type, int8_t level); static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid); static void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, @@ -600,17 +600,6 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { return 0; } -static void tdDestroySDataBlockArray(SArray *pArray) { - // TODO -#if 0 - for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SSDataBlock *pDataBlock = taosArrayGet(pArray, i); - blockDestroyInner(pDataBlock); - } -#endif - taosArrayDestroy(pArray); -} - /** * @brief retention of rsma1/rsma2 * @@ -668,8 +657,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm } else { smaDebug("vgId:%d, rsma %" PRIi8 " data fetched", SMA_VID(pSma), pItem->level); } - -#if 1 +#if 0 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, pItem->level); blockDebugShowDataBlocks(pResList, flag); @@ -731,11 +719,45 @@ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inpu taosWriteQitem(pInfo->queue, qItem); SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); - atomic_fetch_add_64(&pRSmaStat->qBufSize, taosQueueMemorySize(pInfo->queue)); + int64_t bufSize = atomic_add_fetch_64(&pRSmaStat->qBufSize, pReq->header.contLen); + + // smoothing consume + int32_t n = bufSize / RSMA_QTASKEXEC_BUFSIZ; + if (n > 1) { + if (n > 10) { + n = 10; + } + taosMsleep(n << 4); + if (n > 2) { + smaWarn("vgId:%d pInfo->queue itemSize:%d, memSize:%" PRIi64 ", sleep %d ms", SMA_VID(pSma), + taosQueueItemSize(pInfo->queue), taosQueueMemorySize(pInfo->queue), n << 4); + } else { + smaDebug("vgId:%d pInfo->queue itemSize:%d, memSize:%" PRIi64 ", sleep %d ms", SMA_VID(pSma), + taosQueueItemSize(pInfo->queue), taosQueueMemorySize(pInfo->queue), n << 4); + } + } return TSDB_CODE_SUCCESS; } +static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { + SSubmitMsgIter msgIter = {0}; + SSubmitBlkIter blkIter = {0}; + STSRow *row = NULL; + if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1; + while (true) { + SSubmitBlk *pBlock = NULL; + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + if (pBlock == NULL) break; + tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); + while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { + smaDebug("vgId:%d numOfRows:%d, suid:%" PRIi64 ", uid:%" PRIi64 ", version:%" PRIi64 ", ts:%" PRIi64, + SMA_VID(pSma), msgIter.numOfRows, msgIter.suid, msgIter.uid, pReq->version, row->ts); + } + } + return 0; +} + /** * @brief sync mode * @@ -744,32 +766,42 @@ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inpu * @param msgSize * @param inputType * @param pInfo - * @param suid + * @param type * @param level * @return int32_t */ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo, - tb_uid_t suid, int8_t level) { + int8_t type, int8_t level) { int32_t idx = level - 1; - if (!pInfo || !RSMA_INFO_QTASK(pInfo, idx)) { - smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); + + void *qTaskInfo = (type == 0) ? RSMA_INFO_QTASK(pInfo, idx) : RSMA_INFO_IQTASK(pInfo, idx); + if (!qTaskInfo) { + smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, + pInfo->suid); return TSDB_CODE_SUCCESS; } if (!pInfo->pTSchema) { - smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); + smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, pInfo->suid); return TSDB_CODE_FAILED; } smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, - RSMA_INFO_QTASK(pInfo, idx), suid); + RSMA_INFO_QTASK(pInfo, idx), pInfo->suid); - if (qSetMultiStreamInput(RSMA_INFO_QTASK(pInfo, idx), pMsg, msgSize, inputType) < 0) { // INPUT__DATA_SUBMIT +#if 0 + for (int32_t i = 0; i < msgSize; ++i) { + SSubmitReq *pReq = *(SSubmitReq **)((char *)pMsg + i * sizeof(void *)); + smaDebug("vgId:%d [%d][%d] version %" PRIi64, SMA_VID(pSma), msgSize, i, pReq->version); + tdRsmaPrintSubmitReq(pSma, pReq); + } +#endif + if (qSetMultiStreamInput(qTaskInfo, pMsg, msgSize, inputType) < 0) { smaError("vgId:%d, rsma %" PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); - tdRSmaFetchAndSubmitResult(pSma, RSMA_INFO_QTASK(pInfo, idx), pItem, pInfo->pTSchema, suid); + tdRSmaFetchAndSubmitResult(pSma, qTaskInfo, pItem, pInfo->pTSchema, pInfo->suid); atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); if (smaMgmt.tmrHandle) { @@ -858,6 +890,8 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_FAILED; } + } else { + ASSERT(0); } tdReleaseRSmaInfo(pSma, pRSmaInfo); @@ -896,9 +930,18 @@ static int32_t tdRSmaExecCheck(SSma *pSma) { int64_t bufSize = atomic_load_64(&pRsmaStat->qBufSize); if ((pRsmaStat->execStat == 1) || (bufSize < RSMA_QTASKEXEC_BUFSIZ)) { + if (bufSize > RSMA_QTASKEXEC_BUFSIZ) { + smaDebug("vgId:%d bufSize is %d but has no chance to exec as qTaskInfo occupied by another task", SMA_VID(pSma), + bufSize); + } else { + smaDebug("vgId:%d bufSize is %d but has no chance to exec as less than %d", SMA_VID(pSma), bufSize, + RSMA_QTASKEXEC_BUFSIZ); + } return TSDB_CODE_SUCCESS; } + smaDebug("vgId:%d bufSize is %d and has chance to exec as qTaskInfo is free now", SMA_VID(pSma), bufSize); + pRsmaStat->execStat = 1; SRSmaExecMsg fetchMsg; @@ -1633,6 +1676,7 @@ int32_t smaProcessFetch(SSma *pSma, void *pMsg) { goto _err; } +#if 0 pInfo = tdAcquireRSmaInfoBySuid(pSma, req.suid); if (!pInfo) { if (terrno == TSDB_CODE_SUCCESS) { @@ -1657,12 +1701,13 @@ int32_t smaProcessFetch(SSma *pSma, void *pMsg) { tdCleanupStreamInputDataBlock(taskInfo); tdReleaseRSmaInfo(pSma, pInfo); +#endif tDecoderClear(&decoder); smaDebug("vgId:%d, success to process rsma fetch msg for suid:%" PRIi64 " level:%" PRIi8, SMA_VID(pSma), req.suid, req.level); return TSDB_CODE_SUCCESS; _err: - tdReleaseRSmaInfo(pSma, pInfo); + // tdReleaseRSmaInfo(pSma, pInfo); tDecoderClear(&decoder); smaError("vgId:%d, failed to process rsma fetch msg since %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; @@ -1674,7 +1719,14 @@ static void tdFreeRSmaSubmitItems(SArray *pItems) { } } -int32_t tdRSmaProcessExecImpl(SSma *pSma) { +/** + * @brief + * + * @param pSma + * @param type 0 triggered when buffer overflow, 1 triggered by commit + * @return int32_t + */ +int32_t tdRSmaProcessExecImpl(SSma *pSma, int8_t type) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SHashObj *infoHash = NULL; @@ -1686,12 +1738,14 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma) { goto _err; } - taosRLockLatch(SMA_ENV_LOCK(pEnv)); - if (atomic_load_64(&pRSmaStat->qBufSize) < RSMA_QTASKEXEC_BUFSIZ) { + if (type == 0) { + taosRLockLatch(SMA_ENV_LOCK(pEnv)); + if (atomic_load_64(&pRSmaStat->qBufSize) < RSMA_QTASKEXEC_BUFSIZ) { + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + return TSDB_CODE_SUCCESS; + } taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - return TSDB_CODE_SUCCESS; } - taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); if (!(pSubmitQArr = taosArrayInit(taosHashGetSize(infoHash), sizeof(SRSmaExecQItem)))) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1706,18 +1760,33 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma) { SRSmaExecQItem qItem = {0}; taosWLockLatch(SMA_ENV_LOCK(pEnv)); void *pIter = taosHashIterate(infoHash, NULL); - while (pIter) { - SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; - if (taosQueueItemSize(pInfo->queue)) { - taosReadAllQitems(pInfo->queue, pInfo->qall); - qItem.qall = &pInfo->qall; - qItem.pRSmaInfo = pIter; - taosArrayPush(pSubmitQArr, &qItem); + if (type == 0) { + while (pIter) { + SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; + if (taosQueueItemSize(pInfo->queue)) { + taosReadAllQitems(pInfo->queue, pInfo->qall); + qItem.qall = &pInfo->qall; + qItem.pRSmaInfo = pIter; + taosArrayPush(pSubmitQArr, &qItem); + } + ASSERT(taosQueueItemSize(pInfo->queue) == 0); + pIter = taosHashIterate(infoHash, pIter); } - ASSERT(taosQueueItemSize(pInfo->queue) == 0); - pIter = taosHashIterate(infoHash, pIter); + } else if (type == 1) { + while (pIter) { + SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; + if (taosQueueItemSize(pInfo->iQueue)) { + taosReadAllQitems(pInfo->iQueue, pInfo->iQall); + qItem.qall = &pInfo->iQall; + qItem.pRSmaInfo = pIter; + taosArrayPush(pSubmitQArr, &qItem); + } + ASSERT(taosQueueItemSize(pInfo->iQueue) == 0); + pIter = taosHashIterate(infoHash, pIter); + } + } else { + ASSERT(0); } - atomic_store_64(&pRSmaStat->qBufSize, 0); taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); @@ -1739,12 +1808,16 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma) { int32_t size = taosArrayGetSize(pSubmitArr); if (size > 0) { - SRSmaInfo *pInfo = *(SRSmaInfo **)pItem->pRSmaInfo; - for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { - if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, pInfo->suid, i) < 0) { - tdFreeRSmaSubmitItems(pSubmitArr); - goto _err; + if (type == 0 || type == 1) { + SRSmaInfo *pInfo = *(SRSmaInfo **)pItem->pRSmaInfo; + for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { + if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, type, i) < 0) { + tdFreeRSmaSubmitItems(pSubmitArr); + goto _err; + } } + } else { + ASSERT(0); } tdFreeRSmaSubmitItems(pSubmitArr); taosArrayClear(pSubmitArr); @@ -1775,16 +1848,17 @@ int32_t smaProcessExec(SSma *pSma, void *pMsg) { terrno = TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP; goto _err; } - - if (tdRSmaProcessExecImpl(pSma) < 0) { + smaDebug("vgId:%d, begin to process rsma exec msg by thread:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); + if (tdRSmaProcessExecImpl(pSma, 0) < 0) { goto _err; } pRsmaStat->execStat = 0; - smaWarn("vgId:%d, success to process rsma exec msg", SMA_VID(pSma)); + smaDebug("vgId:%d, success to process rsma exec msg by thead:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); return TSDB_CODE_SUCCESS; _err: pRsmaStat->execStat = 0; - smaError("vgId:%d, failed to process rsma fetch msg since %s", SMA_VID(pSma), terrstr()); + smaError("vgId:%d, failed to process rsma fetch msg by thread:%p since %s", SMA_VID(pSma), + (void *)taosGetSelfPthreadId(), terrstr()); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 0623b3bd10..5a22114ab4 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -78,7 +78,7 @@ void vnodeBufPoolReset(SVBufPool *pPool) { void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { SVBufPoolNode *pNode; void *p; - + taosThreadSpinLock(&pPool->lock); if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { // allocate from the anchor node p = pPool->ptr; @@ -89,6 +89,7 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { pNode = taosMemoryMalloc(sizeof(*pNode) + size); if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosThreadSpinUnlock(&pPool->lock); return NULL; } @@ -101,7 +102,7 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { pPool->size = pPool->size + sizeof(*pNode) + size; } - + taosThreadSpinUnlock(&pPool->lock); return p; } @@ -129,6 +130,12 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) return -1; } + if (taosThreadSpinInit(&pPool->lock, 0) != 0) { + taosMemoryFree(pPool); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pPool->next = NULL; pPool->pVnode = pVnode; pPool->nRef = 0; @@ -145,6 +152,7 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) static int vnodeBufPoolDestroy(SVBufPool *pPool) { vnodeBufPoolReset(pPool); + taosThreadSpinDestroy(&pPool->lock); taosMemoryFree(pPool); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index c8dc07af0a..2f5169a0ec 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -220,9 +220,6 @@ int vnodeCommit(SVnode *pVnode) { vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64, TD_VID(pVnode), pVnode->state.commitID, pVnode->state.applied); - vnodeBufPoolUnRef(pVnode->inUse); - pVnode->inUse = NULL; - pVnode->state.commitTerm = pVnode->state.applyTerm; // save info @@ -248,7 +245,7 @@ int vnodeCommit(SVnode *pVnode) { } if (VND_IS_RSMA(pVnode)) { - smaAsyncCommit(pVnode->pSma); + smaAsyncCommit(pVnode->pSma); // would write L2/L3 data into BufPool if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { ASSERT(0); @@ -268,6 +265,9 @@ int vnodeCommit(SVnode *pVnode) { return -1; } } + + vnodeBufPoolUnRef(pVnode->inUse); + pVnode->inUse = NULL; if (tqCommit(pVnode->pTq) < 0) { ASSERT(0); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 32d564d502..adb79fc6ad 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1174,6 +1174,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* tsCol = (TSKEY*)pColDataInfo->pData; + bool inserted = updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.uid); for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; @@ -1183,7 +1184,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[rowId], &pInfo->interval, TSDB_ORDER_ASC); isClosed = isCloseWindow(&win, &pInfo->twAggSup); } - bool inserted = updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.uid); + // must check update info first. bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); bool closedWin = isClosed && inserted && isSignleIntervalWindow(pInfo) && From 559286db71a002447b09bfd58d6a9c23c942381e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Aug 2022 07:51:09 +0800 Subject: [PATCH 045/194] refactor(query): support read data from last files. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 241 ++++++++++++++++--------- 1 file changed, 157 insertions(+), 84 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 73454e2a63..48b7502e8f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -28,6 +28,11 @@ typedef struct { bool hasVal; } SIterInfo; +typedef struct { + int32_t numOfBlocks; + int32_t numOfLastBlocks; +} SBlockNumber; + typedef struct STableBlockScanInfo { uint64_t uid; TSKEY lastKey; @@ -71,10 +76,11 @@ typedef struct SBlockLoadSuppInfo { } SBlockLoadSuppInfo; typedef struct SFilesetIter { - int32_t numOfFiles; // number of total files - int32_t index; // current accessed index in the list - SArray* pFileList; // data file list + int32_t numOfFiles; // number of total files + int32_t index; // current accessed index in the list + SArray* pFileList; // data file list int32_t order; + SArray* pLastBlockList;// last block array list } SFilesetIter; typedef struct SFileDataBlockInfo { @@ -84,12 +90,14 @@ typedef struct SFileDataBlockInfo { } SFileDataBlockInfo; typedef struct SDataBlockIter { - int32_t numOfBlocks; + SBlockNumber numOfBlocks; int32_t index; - SArray* blockList; // SArray + SArray* blockList; // SArray + SArray* pLastBlockList; // last block list int32_t order; - SBlock block; // current SBlock data + SBlock block; // current SBlock data SHashObj* pTableMap; + SArray* pBlockL; // ptr to SBlockL in fileIterator } SDataBlockIter; typedef struct SFileBlockDumpInfo { @@ -132,6 +140,7 @@ struct STsdbReader { STSchema* pSchema; SDataFReader* pFileReader; SVersionRange verRange; + SArray* pLastBlock; // last block info int32_t step; STsdbReader* innerReader[2]; @@ -299,6 +308,13 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, int32 pIter->pFileList = aDFileSet; pIter->numOfFiles = numOfFileset; + pIter->pLastBlockList = taosArrayInit(4, sizeof(SBlockL)); + if (pIter->pLastBlockList == NULL) { + int32_t code = TSDB_CODE_OUT_OF_MEMORY; + tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), idstr); + return code; + } + tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, idstr); return TSDB_CODE_SUCCESS; } @@ -359,7 +375,7 @@ _err: static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, SHashObj* pTableMap) { pIter->order = order; pIter->index = -1; - pIter->numOfBlocks = -1; + pIter->numOfBlocks = (SBlockNumber){-1, -1}; if (pIter->blockList == NULL) { pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); } else { @@ -592,10 +608,10 @@ _end: return code; } -static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_t* numOfValidTables, - int32_t* numOfBlocks) { +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* pLastBlockIndex, + SBlockNumber * pBlockNum, SArray* pQualifiedLastBlock) { + int32_t numOfQTable= 0; size_t numOfTables = taosArrayGetSize(pIndexList); - *numOfValidTables = 0; int64_t st = taosGetTimestampUs(); size_t size = 0; @@ -640,19 +656,44 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_ return TSDB_CODE_OUT_OF_MEMORY; } - (*numOfBlocks) += 1; + pBlockNum->numOfBlocks += 1; } if (pScanInfo->pBlockList != NULL && taosArrayGetSize(pScanInfo->pBlockList) > 0) { - (*numOfValidTables) += 1; + numOfQTable += 1; } } - double el = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("load block of %d tables completed, blocks:%d in %d tables, size:%.2f Kb, elapsed time:%.2f ms %s", - numOfTables, *numOfBlocks, *numOfValidTables, size / 1000.0, el, pReader->idStr); + size_t numOfLast = taosArrayGetSize(pLastBlockIndex); + for(int32_t i = 0; i < numOfLast; ++i) { + SBlockL* pLastBlock = taosArrayGet(pLastBlockIndex, i); + if (pLastBlock->suid != pReader->suid) { + continue; + } - pReader->cost.numOfBlocks += (*numOfBlocks); + { + // 1. time range check, todo add later +// if (pLastBlock->.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { +// continue; +// } + + // 2. version range check + if (pLastBlock->minVer > pReader->verRange.maxVer || pLastBlock->maxVer < pReader->verRange.minVer) { + continue; + } + + pBlockNum->numOfLastBlocks += 1; + taosArrayPush(pQualifiedLastBlock, &i); + } + } + + int32_t total = pBlockNum->numOfLastBlocks + pBlockNum->numOfBlocks; + + double el = (taosGetTimestampUs() - st) / 1000.0; + tsdbDebug("load block of %d tables completed, blocks:%d in %d tables, lastBlock:%d, size:%.2f Kb, elapsed time:%.2f ms %s", + numOfTables, total, numOfQTable, pBlockNum->numOfLastBlocks, size / 1000.0, el, pReader->idStr); + + pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; return TSDB_CODE_SUCCESS; @@ -682,8 +723,12 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ } static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { - SFileDataBlockInfo* pFBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); - return pFBlockInfo; + if (pBlockIter->index >= pBlockIter->numOfBlocks.numOfBlocks) { + return NULL; + } else { + SFileDataBlockInfo* pFBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); + return pFBlockInfo; + } } static SBlock* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } @@ -771,41 +816,45 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn return TSDB_CODE_SUCCESS; } -static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, - STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { +static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData) { int64_t st = taosGetTimestampUs(); + double elapsedTime = 0; + int32_t code = 0; SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - SBlock* pBlock = getCurrentBlock(pBlockIter); - - SSDataBlock* pResBlock = pReader->pResBlock; - int32_t numOfCols = blockDataGetNumOfCols(pResBlock); - - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBlockIdx blockIdx = {.suid = pReader->suid, .uid = pBlockScanInfo->uid}; - int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData /*, pSupInfo->colIds, numOfCols*/); - if (code != TSDB_CODE_SUCCESS) { - goto _error; + if (pFBlock != NULL) { + SBlock* pBlock = getCurrentBlock(pBlockIter); + code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + + tsdbDebug("%p load file block into buffer, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 + ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", + pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, + pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); + } else { + int32_t pos = pBlockIter->index - pBlockIter->numOfBlocks.numOfBlocks; + int32_t* index = taosArrayGet(pBlockIter->pLastBlockList, pos); + + SBlockL* pBlockL = taosArrayGet(pReader->status.fileIter.pLastBlockList, *index); + code = tsdbReadLastBlock(pReader->pFileReader, pBlockL, pBlockData); } - double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; pReader->cost.blockLoadTime += elapsedTime; - pDumpInfo->allDumped = false; - tsdbDebug("%p load file block into buffer, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 - ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", - pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, - pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; _error: - tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 - ", rows:%d, %s", - pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, - pReader->idStr); +// tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 +// ", rows:%d, %s", +// pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, +// pReader->idStr); return code; } @@ -861,10 +910,11 @@ static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, v static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) { SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); - - int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx); - tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetBlock); + if (pFBlock != NULL) { + STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); + int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx); + tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetBlock); + } #if 0 qDebug("check file block, table uid:%"PRIu64" index:%d offset:%"PRId64", ", pScanInfo->uid, *mapDataIndex, pBlockIter->block.aSubBlock[0].offset); @@ -873,12 +923,17 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) { return TSDB_CODE_SUCCESS; } -static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks) { +static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockNumber* pBlockNum, SArray* pQLastBlock) { bool asc = ASCENDING_TRAVERSE(pReader->order); - pBlockIter->numOfBlocks = numOfBlocks; + pBlockIter->numOfBlocks = *pBlockNum; taosArrayClear(pBlockIter->blockList); + if (pBlockNum->numOfLastBlocks > 0) { + taosArrayDestroy(pBlockIter->pLastBlockList); + pBlockIter->pLastBlockList = pQLastBlock; + } + // access data blocks according to the offset of each block in asc/desc order. int32_t numOfTables = (int32_t)taosHashGetSize(pReader->status.pTableMap); @@ -930,20 +985,21 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte sup.numOfTables += 1; } - ASSERT(numOfBlocks == cnt); + ASSERT(pBlockNum->numOfBlocks == cnt); + int32_t total = pBlockNum->numOfLastBlocks + pBlockNum->numOfBlocks; // since there is only one table qualified, blocks are not sorted - if (sup.numOfTables == 1) { - for (int32_t i = 0; i < numOfBlocks; ++i) { + if (sup.numOfTables == 1 || pBlockNum->numOfBlocks == 0) { + for (int32_t i = 0; i < pBlockNum->numOfBlocks; ++i) { SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i}; taosArrayPush(pBlockIter->blockList, &blockInfo); } int64_t et = taosGetTimestampUs(); tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s", - pReader, cnt, (et - st) / 1000.0, pReader->idStr); + pReader, total, (et - st) / 1000.0, pReader->idStr); - pBlockIter->index = asc ? 0 : (numOfBlocks - 1); + pBlockIter->index = asc ? 0 : (total - 1); cleanupBlockOrderSupporter(&sup); doSetCurrentBlock(pBlockIter); return TSDB_CODE_SUCCESS; @@ -952,7 +1008,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables, pReader->idStr); - assert(cnt <= numOfBlocks && sup.numOfTables <= numOfTables); + ASSERT(cnt <= pBlockNum->numOfBlocks && sup.numOfTables <= numOfTables); SMultiwayMergeTreeInfo* pTree = NULL; uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); @@ -979,12 +1035,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } int64_t et = taosGetTimestampUs(); - tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, cnt, (et - st) / 1000.0, + tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, total, (et - st) / 1000.0, pReader->idStr); cleanupBlockOrderSupporter(&sup); taosMemoryFree(pTree); - pBlockIter->index = asc ? 0 : (numOfBlocks - 1); + pBlockIter->index = asc ? 0 : (total - 1); doSetCurrentBlock(pBlockIter); return TSDB_CODE_SUCCESS; @@ -994,7 +1050,7 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) { bool asc = ASCENDING_TRAVERSE(pBlockIter->order); int32_t step = asc ? 1 : -1; - if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { + if ((pBlockIter->index >= pBlockIter->numOfBlocks.numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { return false; } @@ -1041,7 +1097,7 @@ static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlock int32_t step = ASCENDING_TRAVERSE(pBlockIter->order) ? 1 : -1; int32_t index = pBlockIter->index; - while (index < pBlockIter->numOfBlocks && index >= 0) { + while (index < pBlockIter->numOfBlocks.numOfBlocks && index >= 0) { SFileDataBlockInfo* pFBlock = taosArrayGet(pBlockIter->blockList, index); if (pFBlock->uid == pFBlockInfo->uid && pFBlock->tbBlockIdx == pFBlockInfo->tbBlockIdx) { return index; @@ -1055,7 +1111,7 @@ static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlock } static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t index, int32_t step) { - if (index < 0 || index >= pBlockIter->numOfBlocks) { + if (index < 0 || index >= pBlockIter->numOfBlocks.numOfBlocks) { return -1; } @@ -1473,9 +1529,12 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } } -static int32_t buildComposedDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { +static int32_t buildComposedDataBlock(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->pResBlock; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + STableBlockScanInfo* pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; @@ -1696,7 +1755,7 @@ static TSDBKEY getCurrentKeyInBuf(SDataBlockIter* pBlockIter, STsdbReader* pRead return key; } -static int32_t moveToNextFile(STsdbReader* pReader, int32_t* numOfBlocks) { +static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pQLastBlock) { SReaderStatus* pStatus = &pReader->status; size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); @@ -1715,18 +1774,25 @@ static int32_t moveToNextFile(STsdbReader* pReader, int32_t* numOfBlocks) { return code; } - if (taosArrayGetSize(pIndexList) > 0) { - uint32_t numOfValidTable = 0; - code = doLoadFileBlock(pReader, pIndexList, &numOfValidTable, numOfBlocks); + code = tsdbReadBlockL(pReader->pFileReader, pStatus->fileIter.pLastBlockList); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pIndexList); + return code; + } + + if (taosArrayGetSize(pIndexList) > 0 || taosArrayGetSize(pStatus->fileIter.pLastBlockList) > 0) { + code = doLoadFileBlock(pReader, pIndexList, pStatus->fileIter.pLastBlockList, pBlockNum, pQLastBlock); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; } - if (numOfValidTable > 0) { + if (pBlockNum->numOfBlocks + pBlockNum->numOfLastBlocks > 0) { + ASSERT(taosArrayGetSize(pQLastBlock) == pBlockNum->numOfLastBlocks); break; } } + // no blocks in current file, try next files } @@ -1741,21 +1807,27 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SDataBlockIter* pBlockIter = &pStatus->blockIter; SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - STableBlockScanInfo* pScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); + STableBlockScanInfo* pScanInfo = NULL; + SBlock* pBlock = NULL; + TSDBKEY key = {0}; - SBlock* pBlock = getCurrentBlock(pBlockIter); + if (pFBlock != NULL) { + pScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); + pBlock = getCurrentBlock(pBlockIter); + key = getCurrentKeyInBuf(pBlockIter, pReader); + } - TSDBKEY key = getCurrentKeyInBuf(pBlockIter, pReader); - if (fileBlockShouldLoad(pReader, pFBlock, pBlock, pScanInfo, key)) { + if (pFBlock == NULL || fileBlockShouldLoad(pReader, pFBlock, pBlock, pScanInfo, key)) { tBlockDataReset(&pStatus->fileBlockData); tBlockDataClear(&pStatus->fileBlockData); - code = doLoadFileBlockData(pReader, pBlockIter, pScanInfo, &pStatus->fileBlockData); + + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { return code; } // build composed data block - code = buildComposedDataBlock(pReader, pScanInfo); + code = buildComposedDataBlock(pReader); } else if (bufferDataInFileBlockGap(pReader->order, key, pBlock)) { // data in memory that are earlier than current file block // todo rows in buffer should be less than the file block in asc, greater than file block in desc @@ -1819,20 +1891,23 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) } static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - int32_t numOfBlocks = 0; - int32_t code = moveToNextFile(pReader, &numOfBlocks); + SBlockNumber num = {0}; + SArray* pQLastBlock = taosArrayInit(4, sizeof(int32_t)); + + int32_t code = moveToNextFile(pReader, &num, pQLastBlock); if (code != TSDB_CODE_SUCCESS) { return code; } // all data files are consumed, try data in buffer - if (numOfBlocks == 0) { + if (num.numOfBlocks + num.numOfLastBlocks == 0) { + taosArrayDestroy(pQLastBlock); pReader->status.loadFromFile = false; return code; } // initialize the block iterator for a new fileset - code = initBlockIterator(pReader, pBlockIter, numOfBlocks); + code = initBlockIterator(pReader, pBlockIter, &num, pQLastBlock); // set the correct start position according to the query time window initBlockDumpInfo(pReader, pBlockIter); @@ -1851,13 +1926,10 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { SDataBlockIter* pBlockIter = &pReader->status.blockIter; while (1) { - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); - STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; if (fileBlockPartiallyRead(pDumpInfo, asc)) { // file data block is partially loaded - code = buildComposedDataBlock(pReader, pScanInfo); + code = buildComposedDataBlock(pReader); } else { // current block are exhausted, try the next file block if (pDumpInfo->allDumped) { @@ -2209,7 +2281,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn // 3. load the neighbor block, and set it to be the currently accessed file data block tBlockDataReset(&pStatus->fileBlockData); tBlockDataClear(&pStatus->fileBlockData); - int32_t code = doLoadFileBlockData(pReader, pBlockIter, pScanInfo, &pStatus->fileBlockData); + int32_t code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2569,6 +2641,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl pCond->order = TSDB_ORDER_ASC; } + // here we only need one more row, so the capacity is set to be ONE. code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -2872,7 +2945,7 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { tBlockDataReset(&pStatus->fileBlockData); tBlockDataClear(&pStatus->fileBlockData); - int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, pBlockScanInfo, &pStatus->fileBlockData); + int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData, 1); @@ -2968,12 +3041,12 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa SDataBlockIter* pBlockIter = &pStatus->blockIter; pTableBlockInfo->numOfFiles += pStatus->fileIter.numOfFiles; - if (pBlockIter->numOfBlocks > 0) { - pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; + if (pBlockIter->numOfBlocks.numOfBlocks > 0) { + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks.numOfBlocks; } pTableBlockInfo->numOfTables = numOfTables; - bool hasNext = (pBlockIter->numOfBlocks > 0); + bool hasNext = (pBlockIter->numOfBlocks.numOfBlocks > 0); while (true) { if (hasNext) { @@ -3004,8 +3077,8 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa break; } - pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; - hasNext = (pBlockIter->numOfBlocks > 0); + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks.numOfBlocks; + hasNext = (pBlockIter->numOfBlocks.numOfBlocks > 0); } // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, From b888c90054fb5f82479f93c6fafe9b577f8598e1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 17 Aug 2022 10:38:40 +0800 Subject: [PATCH 046/194] fix: change the interval data load status when scan --- source/libs/executor/src/scanoperator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 454a0b0070..12df378a42 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -139,7 +139,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } assert(w.ekey > pBlockInfo->window.ekey); - if (w.skey <= pBlockInfo->window.ekey && w.skey > pBlockInfo->window.skey) { + if (TMAX(w.skey, pBlockInfo->window.skey) <= pBlockInfo->window.ekey) { return true; } } @@ -147,7 +147,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey); assert(w.skey <= pBlockInfo->window.ekey); - if (w.skey > pBlockInfo->window.skey) { + if (TMAX(w.skey, pBlockInfo->window.skey) <= TMIN(w.ekey, pBlockInfo->window.ekey)) { return true; } @@ -158,7 +158,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } assert(w.skey < pBlockInfo->window.skey); - if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { + if (pBlockInfo->window.skey <= TMIN(w.ekey, pBlockInfo->window.ekey)) { return true; } } From 23a0af9d1272cbfe5efa150e84cd213c878d7089 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Aug 2022 11:46:13 +0800 Subject: [PATCH 047/194] more work --- source/dnode/vnode/src/inc/tsdb.h | 16 +++------------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 ++++ source/dnode/vnode/src/tsdb/tsdbUtil.c | 24 ++++++++++++++---------- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 7e88a884c5..88960b5706 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -302,13 +302,6 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode); int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); // structs ======================= -typedef struct { - int minFid; - int midFid; - int maxFid; - TSKEY minKey; -} SRtn; - struct STsdbFS { SDelFile *pDelFile; SArray *aDFileSet; // SArray @@ -336,19 +329,14 @@ struct SMemSkipListNode { SMemSkipListNode *forwards[0]; }; typedef struct SMemSkipList { - uint32_t seed; int64_t size; + uint32_t seed; int8_t maxLevel; int8_t level; SMemSkipListNode *pHead; SMemSkipListNode *pTail; } SMemSkipList; -struct SDelDataInfo { - tb_uid_t suid; - tb_uid_t uid; -}; - struct STbData { tb_uid_t suid; tb_uid_t uid; @@ -448,6 +436,8 @@ struct SBlockL { int64_t suid; int64_t minUid; int64_t maxUid; + TSKEY minKey; + TSKEY maxKey; int64_t minVer; int64_t maxVer; int32_t nRow; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 589bce9aa5..7374988fc9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -567,9 +567,13 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { // info blockL.suid = pBlockData->suid; blockL.nRow = pBlockData->nRow; + blockL.minKey = TSKEY_MAX; + blockL.maxKey = TSKEY_MIN; blockL.minVer = VERSION_MAX; blockL.maxVer = VERSION_MIN; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + blockL.minKey = TMIN(blockL.minKey, pBlockData->aTSKEY[iRow]); + blockL.maxKey = TMAX(blockL.maxKey, pBlockData->aTSKEY[iRow]); blockL.minVer = TMIN(blockL.minVer, pBlockData->aVersion[iRow]); blockL.maxVer = TMAX(blockL.maxVer, pBlockData->aVersion[iRow]); } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 897c2359fa..e890c624ec 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -284,6 +284,8 @@ int32_t tPutBlockL(uint8_t *p, void *ph) { n += tPutI64(p ? p + n : p, pBlockL->suid); n += tPutI64(p ? p + n : p, pBlockL->minUid); n += tPutI64(p ? p + n : p, pBlockL->maxUid); + n += tPutI64v(p ? p + n : p, pBlockL->minKey); + n += tPutI64v(p ? p + n : p, pBlockL->maxKey); n += tPutI64v(p ? p + n : p, pBlockL->minVer); n += tPutI64v(p ? p + n : p, pBlockL->maxVer); n += tPutI32v(p ? p + n : p, pBlockL->nRow); @@ -301,6 +303,8 @@ int32_t tGetBlockL(uint8_t *p, void *ph) { n += tGetI64(p + n, &pBlockL->suid); n += tGetI64(p + n, &pBlockL->minUid); n += tGetI64(p + n, &pBlockL->maxUid); + n += tGetI64v(p + n, &pBlockL->minKey); + n += tGetI64v(p + n, &pBlockL->maxKey); n += tGetI64v(p + n, &pBlockL->minVer); n += tGetI64v(p + n, &pBlockL->maxVer); n += tGetI32v(p + n, &pBlockL->nRow); @@ -1651,25 +1655,25 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { break; } case TSDB_DATA_TYPE_FLOAT: { - *(double*)(&pColAgg->sum) += colVal.value.f; - if (!minAssigned || *(double*)(&pColAgg->min) > colVal.value.f) { - *(double*)(&pColAgg->min) = colVal.value.f; + *(double *)(&pColAgg->sum) += colVal.value.f; + if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.f) { + *(double *)(&pColAgg->min) = colVal.value.f; minAssigned = true; } - if (!maxAssigned || *(double*)(&pColAgg->max) < colVal.value.f) { - *(double*)(&pColAgg->max) = colVal.value.f; + if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.f) { + *(double *)(&pColAgg->max) = colVal.value.f; maxAssigned = true; } break; } case TSDB_DATA_TYPE_DOUBLE: { - *(double*)(&pColAgg->sum) += colVal.value.d; - if (!minAssigned || *(double*)(&pColAgg->min) > colVal.value.d) { - *(double*)(&pColAgg->min) = colVal.value.d; + *(double *)(&pColAgg->sum) += colVal.value.d; + if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.d) { + *(double *)(&pColAgg->min) = colVal.value.d; minAssigned = true; } - if (!maxAssigned || *(double*)(&pColAgg->max) < colVal.value.d) { - *(double*)(&pColAgg->max) = colVal.value.d; + if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.d) { + *(double *)(&pColAgg->max) = colVal.value.d; maxAssigned = true; } break; From c7e778f9a842e7ce5d7066aa129eda71d789b249 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 17 Aug 2022 13:30:37 +0800 Subject: [PATCH 048/194] refactor code --- source/libs/transport/src/thttp.c | 164 +++++++++++++----------------- 1 file changed, 70 insertions(+), 94 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 62277a7569..52c692d22a 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -14,14 +14,21 @@ */ #define _DEFAULT_SOURCE -#ifdef USE_UV -#include -#endif // clang-format off -#include "zlib.h" +#include #include "thttp.h" #include "taoserror.h" #include "tlog.h" +#include "zlib.h" + +typedef struct SHttpClient { + uv_connect_t conn; + uv_tcp_t tcp; + uv_write_t req; + uv_buf_t* buf; + char* addr; + uint16_t port; +} SHttpClient; static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pHead, int32_t headLen, EHttpCompFlag flag) { @@ -45,7 +52,7 @@ static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pH } } -int32_t taosCompressHttpRport(char* pSrc, int32_t srcLen) { +static int32_t taosCompressHttpRport(char* pSrc, int32_t srcLen) { int32_t code = -1; int32_t destLen = srcLen; void* pDest = taosMemoryMalloc(destLen); @@ -114,40 +121,53 @@ _OVER: return code; } -#ifdef USE_UV -static void clientConnCb(uv_connect_t* req, int32_t status) { - if (status < 0) { +static void destroyHttpClient(SHttpClient* cli) { + taosMemoryFree(cli->buf); + taosMemoryFree(cli->addr); + taosMemoryFree(cli); +} +static void clientCloseCb(uv_handle_t* handle) { + SHttpClient* cli = handle->data; + destroyHttpClient(cli); +} +static void clientSentCb(uv_write_t* req, int32_t status) { + SHttpClient* cli = req->data; + if (status != 0) { terrno = TAOS_SYSTEM_ERROR(status); - uError("connection error %s", uv_strerror(status)); - uv_close((uv_handle_t*)req->handle, NULL); + uError("http-report failed to send data %s", uv_strerror(status)); + } else { + uInfo("http-report succ to send data"); + } + uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); +} +static void clientConnCb(uv_connect_t* req, int32_t status) { + SHttpClient* cli = req->data; + if (status != 0) { + terrno = TAOS_SYSTEM_ERROR(status); + uError("http-report failed to conn to server, reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); + uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); return; } - uv_buf_t* wb = req->data; - assert(wb != NULL); - uv_write_t write_req; - uv_write(&write_req, req->handle, wb, 2, NULL); - uv_close((uv_handle_t*)req->handle, NULL); + uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->buf, 2, clientSentCb); } -int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { +static int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { uint32_t ipv4 = taosGetIpv4FromFqdn(server); if (ipv4 == 0xffffffff) { terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to get http server:%s ip since %s", server, terrstr()); + uError("http-report failed to get http server:%s ip since %s", server, terrstr()); return -1; } - char ipv4Buf[128] = {0}; tinet_ntoa(ipv4Buf, ipv4); - + uv_ip4_addr(ipv4Buf, port, dest); + return 0; +} +int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { struct sockaddr_in dest = {0}; - uv_ip4_addr(ipv4Buf, port, &dest); - - uv_tcp_t socket_tcp = {0}; - uv_loop_t* loop = uv_default_loop(); - uv_tcp_init(loop, &socket_tcp); - uv_connect_t* connect = (uv_connect_t*)taosMemoryMalloc(sizeof(uv_connect_t)); - + if (taosBuildDstAddr(server, port, &dest) < 0) { + return -1; + } if (flag == HTTP_GZIP) { int32_t dstLen = taosCompressHttpRport(pCont, contLen); if (dstLen > 0) { @@ -156,81 +176,37 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 flag = HTTP_FLAT; } } + terrno = 0; char header[1024] = {0}; int32_t headLen = taosBuildHttpHeader(server, contLen, header, sizeof(header), flag); - uv_buf_t wb[2]; - wb[0] = uv_buf_init((char*)header, headLen); - wb[1] = uv_buf_init((char*)pCont, contLen); + uv_buf_t* wb = taosMemoryCalloc(2, sizeof(uv_buf_t)); + wb[0] = uv_buf_init((char*)header, headLen); // stack var + wb[1] = uv_buf_init((char*)pCont, contLen); // heap var + + SHttpClient* cli = taosMemoryCalloc(1, sizeof(SHttpClient)); + cli->conn.data = cli; + cli->tcp.data = cli; + cli->req.data = cli; + cli->buf = wb; + cli->addr = tstrdup(server); + cli->port = port; + + uv_loop_t* loop = uv_default_loop(); + uv_tcp_init(loop, &cli->tcp); + // set up timeout to avoid stuck; + int32_t fd = taosCreateSocketWithTimeout(5); + uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); + + int32_t ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&dest, clientConnCb); + if (ret != 0) { + uError("http-report failed to connect to server, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); + destroyHttpClient(cli); + } - connect->data = wb; - terrno = 0; - uv_tcp_connect(connect, &socket_tcp, (const struct sockaddr*)&dest, clientConnCb); uv_run(loop, UV_RUN_DEFAULT); uv_loop_close(loop); - taosMemoryFree(connect); return terrno; } - -#else -int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { - int32_t code = -1; - TdSocketPtr pSocket = NULL; - - uint32_t ip = taosGetIpv4FromFqdn(server); - if (ip == 0xffffffff) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to get http server:%s ip since %s", server, terrstr()); - goto SEND_OVER; - } - - pSocket = taosOpenTcpClientSocket(ip, port, 0); - if (pSocket == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to create http socket to %s:%u since %s", server, port, terrstr()); - goto SEND_OVER; - } - - if (flag == HTTP_GZIP) { - int32_t dstLen = taosCompressHttpRport(pCont, contLen); - if (dstLen > 0) { - contLen = dstLen; - } else { - flag = HTTP_FLAT; - } - } - - char header[1024] = {0}; - int32_t headLen = taosBuildHttpHeader(server, contLen, header, sizeof(header), flag); - if (taosWriteMsg(pSocket, header, headLen) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to send http header to %s:%u since %s", server, port, terrstr()); - goto SEND_OVER; - } - - if (taosWriteMsg(pSocket, (void*)pCont, contLen) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to send http content to %s:%u since %s", server, port, terrstr()); - goto SEND_OVER; - } - - // read something to avoid nginx error 499 - if (taosWriteMsg(pSocket, header, 10) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to receive response from %s:%u since %s", server, port, terrstr()); - goto SEND_OVER; - } - - code = 0; - -SEND_OVER: - if (pSocket != NULL) { - taosCloseSocket(&pSocket); - } - - return code; -} - // clang-format on -#endif From c6106be5da57caa6f62891ab66a8303ded35bfeb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Aug 2022 15:08:44 +0800 Subject: [PATCH 049/194] refact more work --- source/dnode/vnode/src/inc/tsdb.h | 5 -- source/dnode/vnode/src/meta/metaQuery.c | 67 ++++++++++++++++++---- source/dnode/vnode/src/tsdb/tsdbCommit.c | 22 +++---- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 15 ----- 4 files changed, 65 insertions(+), 44 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 88960b5706..102b714c1e 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -342,9 +342,6 @@ struct STbData { tb_uid_t uid; TSKEY minKey; TSKEY maxKey; - int64_t minVersion; - int64_t maxVersion; - int32_t maxSkmVer; SDelData *pHead; SDelData *pTail; SMemSkipList sl; @@ -358,8 +355,6 @@ struct SMemTable { volatile int32_t nRef; TSKEY minKey; TSKEY maxKey; - int64_t minVersion; - int64_t maxVersion; int64_t nRow; int64_t nDel; struct { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 6ff55d2f4e..3ce7826701 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -429,18 +429,65 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { } int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sver, STSchema **ppTSchema) { - int32_t code = 0; - STSchema *pTSchema = NULL; - SSkmDbKey skmDbKey = {.uid = suid ? suid : uid, .sver = sver}; + int32_t code = 0; + void *pData = NULL; int nData = 0; + SSkmDbKey skmDbKey; + if (sver <= 0) { + SMetaInfo info; + if (metaGetInfo(pMeta, suid ? suid : uid, &info) == 0) { + sver = info.skmVer; + } else { + TBC *pSkmDbC = NULL; + int c; - // query + skmDbKey.uid = suid ? suid : uid; + skmDbKey.sver = INT32_MAX; + + tdbTbcOpen(pMeta->pSkmDb, &pSkmDbC, NULL); + metaRLock(pMeta); + + if (tdbTbcMoveTo(pSkmDbC, &skmDbKey, sizeof(skmDbKey), &c) < 0) { + metaULock(pMeta); + tdbTbcClose(pSkmDbC); + code = TSDB_CODE_NOT_FOUND; + goto _exit; + } + + ASSERT(c); + + if (c < 0) { + tdbTbcMoveToPrev(pSkmDbC); + } + + const void *pKey = NULL; + int32_t nKey = 0; + tdbTbcGet(pSkmDbC, &pKey, &nKey, NULL, NULL); + + if (((SSkmDbKey *)pKey)->uid != uid) { + metaULock(pMeta); + tdbTbcClose(pSkmDbC); + code = TSDB_CODE_NOT_FOUND; + goto _exit; + } + + sver = ((SSkmDbKey *)pKey)->sver; + + metaULock(pMeta); + tdbTbcClose(pSkmDbC); + } + } + + ASSERT(sver > 0); + + skmDbKey.uid = suid ? suid : uid; + skmDbKey.sver = sver; metaRLock(pMeta); - if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), &pData, &nData) < 0) { - code = TSDB_CODE_NOT_FOUND; + if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(SSkmDbKey), &pData, &nData) < 0) { metaULock(pMeta); - goto _err; + code = TSDB_CODE_NOT_FOUND; + goto _exit; } metaULock(pMeta); @@ -462,15 +509,13 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv SSchema *pSchema = pSchemaWrapper->pSchema + i; tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); } - pTSchema = tdGetSchemaFromBuilder(&sb); + STSchema *pTSchema = tdGetSchemaFromBuilder(&sb); tdDestroyTSchemaBuilder(&sb); *ppTSchema = pTSchema; taosMemoryFree(pSchemaWrapper->pSchema); - return code; -_err: - *ppTSchema = NULL; +_exit: return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 7374988fc9..020f3b0bc6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -290,23 +290,19 @@ _err: return code; } -static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { +static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid) { int32_t code = 0; - if (pCommitter->skmTable.pTSchema) { - if (pCommitter->skmTable.suid == suid) { - if (suid == 0) { - if (pCommitter->skmTable.uid == uid && sver == pCommitter->skmTable.pTSchema->version) goto _exit; - } else { - if (sver == pCommitter->skmTable.pTSchema->version) goto _exit; - } - } + if (suid) { + if (pCommitter->skmTable.suid == suid) goto _exit; + } else { + if (pCommitter->skmTable.uid == uid) goto _exit; } pCommitter->skmTable.suid = suid; pCommitter->skmTable.uid = uid; tTSchemaDestroy(pCommitter->skmTable.pTSchema); - code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmTable.pTSchema); + code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, -1, &pCommitter->skmTable.pTSchema); if (code) goto _exit; _exit: @@ -360,7 +356,7 @@ static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { int64_t suid = pBlockL->suid; int64_t uid = pBlockL->maxUid; - code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid, 1 /*TODO*/); + code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid); if (code) goto _exit; code = tBlockDataInit(pBlockDatal, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); @@ -978,7 +974,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { pBlock = NULL; } - code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid, pTbData->maxSkmVer /*TODO*/); + code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid); if (code) goto _err; tMapDataReset(&pCommitter->dWriter.mBlock); @@ -1160,7 +1156,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { // set block data schema if need if (pBlockDataW->suid == 0 && pBlockDataW->uid == 0) { - code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid, 1 /*TOOD*/); + code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid); if (code) goto _err; code = tBlockDataInit(pBlockDataW, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 803b32517a..a6628463f8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -46,8 +46,6 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable->nRef = 1; pMemTable->minKey = TSKEY_MAX; pMemTable->maxKey = TSKEY_MIN; - pMemTable->minVersion = VERSION_MAX; - pMemTable->maxVersion = VERSION_MIN; pMemTable->nRow = 0; pMemTable->nDel = 0; pMemTable->nTbData = 0; @@ -180,10 +178,6 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pTbData->pTail = pDelData; } - // update the state of pMemTable and other (todo) - - pMemTable->minVersion = TMIN(pMemTable->minVersion, version); - pMemTable->maxVersion = TMAX(pMemTable->maxVersion, version); pMemTable->nDel++; if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && tsdbKeyCmprFn(&lastKey, &pTbData->maxKey) >= 0) { @@ -368,9 +362,6 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid pTbData->uid = uid; pTbData->minKey = TSKEY_MAX; pTbData->maxKey = TSKEY_MIN; - pTbData->minVersion = VERSION_MAX; - pTbData->maxVersion = VERSION_MIN; - pTbData->maxSkmVer = -1; pTbData->pHead = NULL; pTbData->pTail = NULL; pTbData->sl.seed = taosRand(); @@ -615,15 +606,9 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb); } - pTbData->minVersion = TMIN(pTbData->minVersion, version); - pTbData->maxVersion = TMAX(pTbData->maxVersion, version); - pTbData->maxSkmVer = TMAX(pTbData->maxSkmVer, pMsgIter->sversion); - // SMemTable pMemTable->minKey = TMIN(pMemTable->minKey, pTbData->minKey); pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey); - pMemTable->minVersion = TMIN(pMemTable->minVersion, pTbData->minVersion); - pMemTable->maxVersion = TMAX(pMemTable->maxVersion, pTbData->maxVersion); pMemTable->nRow += nRow; pRsp->numOfRows = nRow; From 4e9ead32aaf3693fc46cf4d2a40d7102951a8046 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 17 Aug 2022 15:56:14 +0800 Subject: [PATCH 050/194] refactor code --- source/libs/transport/src/thttp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 52c692d22a..806a786b00 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -196,8 +196,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 uv_loop_t* loop = uv_default_loop(); uv_tcp_init(loop, &cli->tcp); // set up timeout to avoid stuck; - int32_t fd = taosCreateSocketWithTimeout(5); - uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); + //int32_t fd = taosCreateSocketWithTimeout(5); + //uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); int32_t ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&dest, clientConnCb); if (ret != 0) { From 13f5acd4b910f3c13c277d28e0b299bb55736069 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 17 Aug 2022 16:04:58 +0800 Subject: [PATCH 051/194] fix(query): fix interp pResBlock->rows > capacity cause assert failure TD-18445 --- source/libs/executor/src/timewindowoperator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 6418f5305c..3e931a489d 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2154,7 +2154,9 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; - + if (rows >= pResBlock->info.capacity) { + return; + } // todo set the correct primary timestamp column // output the result From 4b47476de1e9ac2301280ab64b06d119c152efeb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 17 Aug 2022 16:51:15 +0800 Subject: [PATCH 052/194] fix: upgrade last files reading for last/last_row caching --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbCache.c | 221 +++++++++++++++++++++--- source/dnode/vnode/src/tsdb/tsdbUtil.c | 21 ++- 3 files changed, 222 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 102b714c1e..6263703c01 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -123,6 +123,7 @@ int32_t tGetBlockL(uint8_t *p, void *ph); int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tCmprBlockIdx(void const *lhs, void const *rhs); +int32_t tCmprBlockL(void const *lhs, void const *rhs); // SColdata void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); void tColDataReset(SColData *pColData); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 24d6b2f385..b515b028b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -279,6 +279,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb } } + _invalidate: taosMemoryFreeClear(pTSchema); taosLRUCacheRelease(pCache, h, invalidate); @@ -404,6 +405,178 @@ _err: return code; } +typedef enum { + SFSLASTNEXTROW_FS, + SFSLASTNEXTROW_FILESET, + SFSLASTNEXTROW_BLOCKDATA, + SFSLASTNEXTROW_BLOCKROW +} SFSLASTNEXTROWSTATES; + +typedef struct { + SFSLASTNEXTROWSTATES state; // [input] + STsdb *pTsdb; // [input] + SBlockIdx *pBlockIdxExp; // [input] + STSchema *pTSchema; // [input] + int32_t nFileSet; + int32_t iFileSet; + SArray *aDFileSet; + SDataFReader *pDataFReader; + SArray *aBlockL; + SBlockL *pBlockL; + SBlockData *pBlockDataL; + SBlockData blockDataL; + int32_t nRow; + int32_t iRow; + TSDBROW row; + /* + SArray *aBlockIdx; + SBlockIdx *pBlockIdx; + SMapData blockMap; + int32_t nBlock; + int32_t iBlock; + SBlock block; + */ +} SFSLastNextRowIter; + +static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { + SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; + int32_t code = 0; + + switch (state->state) { + case SFSLASTNEXTROW_FS: + // state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet; + state->nFileSet = taosArrayGetSize(state->aDFileSet); + state->iFileSet = state->nFileSet; + + state->pBlockDataL = NULL; + + case SFSLASTNEXTROW_FILESET: { + SDFileSet *pFileSet = NULL; + _next_fileset: + if (--state->iFileSet >= 0) { + pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); + } else { + if (state->pBlockDataL) { + tBlockDataDestroy(state->pBlockDataL, 1); + state->pBlockDataL = NULL; + } + + *ppRow = NULL; + return code; + } + + code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet); + if (code) goto _err; + + if (!state->aBlockL) { + state->aBlockL = taosArrayInit(0, sizeof(SBlockIdx)); + } else { + taosArrayClear(state->aBlockL); + } + + code = tsdbReadBlockL(state->pDataFReader, state->aBlockL); + if (code) goto _err; + + // SBlockL *pBlockL = (SBlockL *)taosArrayGet(state->aBlockL, state->iBlockL); + + state->pBlockL = taosArraySearch(state->aBlockL, state->pBlockIdxExp, tCmprBlockL, TD_EQ); + if (!state->pBlockL) { + goto _next_fileset; + } + + int64_t suid = state->pBlockL->suid; + int64_t uid = state->pBlockL->maxUid; + + if (!state->pBlockDataL) { + state->pBlockDataL = &state->blockDataL; + } + code = tBlockDataInit(state->pBlockDataL, suid, suid ? 0 : uid, state->pTSchema); + if (code) goto _err; + } + case SFSLASTNEXTROW_BLOCKDATA: + code = tsdbReadLastBlock(state->pDataFReader, state->pBlockL, state->pBlockDataL); + if (code) goto _err; + + state->nRow = state->blockDataL.nRow; + state->iRow = state->nRow - 1; + + if (!state->pBlockDataL->uid) { + while (state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) { + --state->iRow; + } + } + + state->state = SFSLASTNEXTROW_BLOCKROW; + case SFSLASTNEXTROW_BLOCKROW: + if (state->pBlockDataL->uid) { + if (state->iRow >= 0) { + state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow); + *ppRow = &state->row; + + if (--state->iRow < 0) { + state->state = SFSLASTNEXTROW_FILESET; + } + } + } else { + if (state->iRow >= 0 && state->pBlockIdxExp->uid == state->pBlockDataL->aUid[state->iRow]) { + state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow); + *ppRow = &state->row; + + if (--state->iRow < 0 || state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) { + state->state = SFSLASTNEXTROW_FILESET; + } + } + } + + return code; + default: + ASSERT(0); + break; + } + +_err: + if (state->pDataFReader) { + tsdbDataFReaderClose(&state->pDataFReader); + state->pDataFReader = NULL; + } + if (state->aBlockL) { + taosArrayDestroy(state->aBlockL); + state->aBlockL = NULL; + } + if (state->pBlockDataL) { + tBlockDataDestroy(state->pBlockDataL, 1); + state->pBlockDataL = NULL; + } + + *ppRow = NULL; + + return code; +} + +int32_t clearNextRowFromFSLast(void *iter) { + SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; + int32_t code = 0; + + if (!state) { + return code; + } + + if (state->pDataFReader) { + tsdbDataFReaderClose(&state->pDataFReader); + state->pDataFReader = NULL; + } + if (state->aBlockL) { + taosArrayDestroy(state->aBlockL); + state->aBlockL = NULL; + } + if (state->pBlockDataL) { + tBlockDataDestroy(state->pBlockDataL, 1); + state->pBlockDataL = NULL; + } + + return code; +} + typedef enum SFSNEXTROWSTATES { SFSNEXTROW_FS, SFSNEXTROW_FILESET, @@ -722,18 +895,19 @@ typedef struct { SArray *pSkyline; int64_t iSkyline; - SBlockIdx idx; - SMemNextRowIter memState; - SMemNextRowIter imemState; - SFSNextRowIter fsState; - TSDBROW memRow, imemRow, fsRow; + SBlockIdx idx; + SMemNextRowIter memState; + SMemNextRowIter imemState; + SFSLastNextRowIter fsLastState; + SFSNextRowIter fsState; + TSDBROW memRow, imemRow, fsLastRow, fsRow; - TsdbNextRowState input[3]; + TsdbNextRowState input[4]; STsdbReadSnap *pReadSnap; STsdb *pTsdb; } CacheNextRowIter; -static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb) { +static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema) { int code = 0; tb_uid_t suid = getTableSuidByUid(uid, pTsdb); @@ -779,6 +953,12 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; + pIter->fsLastState.state = SFSNEXTROW_FS; + pIter->fsLastState.pTsdb = pTsdb; + pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; + pIter->fsLastState.pBlockIdxExp = &pIter->idx; + pIter->fsLastState.pTSchema = pTSchema; + pIter->fsState.state = SFSNEXTROW_FS; pIter->fsState.pTsdb = pTsdb; pIter->fsState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; @@ -786,7 +966,9 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, &pIter->memState, getNextRowFromMem, NULL}; pIter->input[1] = (TsdbNextRowState){&pIter->imemRow, true, false, &pIter->imemState, getNextRowFromMem, NULL}; - pIter->input[2] = + pIter->input[2] = (TsdbNextRowState){&pIter->fsLastRow, false, true, &pIter->fsLastState, getNextRowFromFSLast, + clearNextRowFromFSLast}; + pIter->input[3] = (TsdbNextRowState){&pIter->fsRow, false, true, &pIter->fsState, getNextRowFromFS, clearNextRowFromFS}; if (pMem) { @@ -811,7 +993,7 @@ _err: static int32_t nextRowIterClose(CacheNextRowIter *pIter) { int code = 0; - for (int i = 0; i < 3; ++i) { + for (int i = 0; i < 4; ++i) { if (pIter->input[i].nextRowClearFn) { pIter->input[i].nextRowClearFn(pIter->input[i].iter); } @@ -823,7 +1005,6 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) { tsdbUntakeReadSnap(pIter->pTsdb, pIter->pReadSnap); - return code; _err: return code; } @@ -832,7 +1013,7 @@ _err: static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) { int code = 0; - for (int i = 0; i < 3; ++i) { + for (int i = 0; i < 4; ++i) { if (pIter->input[i].next && !pIter->input[i].stop) { code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow); if (code) goto _err; @@ -844,18 +1025,18 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) { } } - if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop) { + if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop && pIter->input[3].stop) { *ppRow = NULL; return code; } - // select maxpoint(s) from mem, imem, fs - TSDBROW *max[3] = {0}; - int iMax[3] = {-1, -1, -1}; + // select maxpoint(s) from mem, imem, fs and last + TSDBROW *max[4] = {0}; + int iMax[4] = {-1, -1, -1, -1}; int nMax = 0; TSKEY maxKey = TSKEY_MIN; - for (int i = 0; i < 3; ++i) { + for (int i = 0; i < 4; ++i) { if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) { TSDBKEY key = TSDBROW_KEY(pIter->input[i].pRow); @@ -873,8 +1054,8 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) { } // delete detection - TSDBROW *merge[3] = {0}; - int iMerge[3] = {-1, -1, -1}; + TSDBROW *merge[4] = {0}; + int iMerge[4] = {-1, -1, -1, -1}; int nMerge = 0; for (int i = 0; i < nMax; ++i) { TSDBKEY maxKey = TSDBROW_KEY(max[i]); @@ -915,7 +1096,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb); + nextRowIterOpen(&iter, uid, pTsdb, pTSchema); do { TSDBROW *pRow = NULL; @@ -1012,7 +1193,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb); + nextRowIterOpen(&iter, uid, pTsdb, pTSchema); do { TSDBROW *pRow = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index e890c624ec..9fcfd9c8bd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -196,6 +196,25 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) { return 0; } +int32_t tCmprBlockL(void const *lhs, void const *rhs) { + SBlockIdx *lBlockIdx = (SBlockIdx *)lhs; + SBlockL *rBlockL = (SBlockL *)rhs; + + if (lBlockIdx->suid < rBlockL->suid) { + return -1; + } else if (lBlockIdx->suid > rBlockL->suid) { + return 1; + } + + if (lBlockIdx->uid < rBlockL->minUid) { + return -1; + } else if (lBlockIdx->uid > rBlockL->maxUid) { + return 1; + } + + return 0; +} + // SBlock ====================================================== void tBlockReset(SBlock *pBlock) { *pBlock = (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; @@ -1944,4 +1963,4 @@ int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t _exit: return code; -} \ No newline at end of file +} From 9eb99615fe58b552ba0ad0222d9e8249b05eb99c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 17 Aug 2022 17:58:46 +0800 Subject: [PATCH 053/194] fix invalid packet --- source/libs/transport/inc/transComm.h | 1 + source/libs/transport/src/transComm.c | 13 +++++++++---- source/libs/transport/src/transSvr.c | 26 ++++++++++++++++++-------- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 6b52c74271..1efb0bb316 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -98,6 +98,7 @@ typedef void* queue[2]; #define TRANS_RETRY_INTERVAL 15 // retry interval (ms) #define TRANS_CONN_TIMEOUT 3 // connect timeout (s) #define TRANS_READ_TIMEOUT 3000 // read timeout (ms) +#define TRANS_PACKET_LIMIT 1024 * 1024 * 512 typedef SRpcMsg STransMsg; typedef SRpcCtx STransCtx; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 4272ec0b1c..af49e5845b 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -112,15 +112,20 @@ int transClearBuffer(SConnBuffer* buf) { } int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) { + static const int HEADSIZE = sizeof(STransMsgHead); + SConnBuffer* p = connBuf; if (p->left != 0) { return -1; } int total = connBuf->total; - *buf = taosMemoryCalloc(1, total); - memcpy(*buf, p->buf, total); - - transResetBuffer(connBuf); + if (total >= HEADSIZE) { + *buf = taosMemoryCalloc(1, total); + memcpy(*buf, p->buf, total); + transResetBuffer(connBuf); + } else { + total = -1; + } return total; } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 3512b27bf8..a94eb01beb 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -184,10 +184,15 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) { } static void uvHandleReq(SSvrConn* pConn) { - STransMsgHead* msg = NULL; - int msgLen = 0; + STrans* pTransInst = pConn->pTransInst; - msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&msg); + STransMsgHead* msg = NULL; + int msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&msg); + if (msgLen <= 0) { + tError("%s conn %p alread read complete packet", transLabel(pTransInst), pConn); + transUnrefSrvHandle(pConn); + return; + } STransMsgHead* pHead = (STransMsgHead*)msg; pHead->code = htonl(pHead->code); @@ -220,7 +225,6 @@ static void uvHandleReq(SSvrConn* pConn) { tDebug("conn %p acquired by server app", pConn); } } - STrans* pTransInst = pConn->pTransInst; STraceId* trace = &pHead->traceId; if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); @@ -268,11 +272,17 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { if (nread > 0) { pBuf->len += nread; tTrace("%s conn %p total read:%d, current read:%d", transLabel(pTransInst), conn, pBuf->len, (int)nread); - while (transReadComplete(pBuf)) { - tTrace("%s conn %p alread read complete packet", transLabel(pTransInst), conn); - uvHandleReq(conn); + if (pBuf->len <= TRANS_PACKET_LIMIT) { + while (transReadComplete(pBuf)) { + tTrace("%s conn %p alread read complete packet", transLabel(pTransInst), conn); + uvHandleReq(conn); + } + return; + } else { + tError("%s conn %p read unexpected packet, exceed limit", transLabel(pTransInst), conn); + destroyConn(conn, true); + return; } - return; } if (nread == 0) { return; From 853e6e29888afcf14ce4a743c9b82f1c19850bb9 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 17 Aug 2022 19:19:58 +0800 Subject: [PATCH 054/194] refactor(mnode): drop stream task --- examples/c/stream_demo.c | 7 +++--- include/common/tcommon.h | 1 + include/libs/stream/tstream.h | 5 +++++ include/util/taoserror.h | 1 + source/dnode/mnode/impl/src/mndScheduler.c | 2 ++ source/dnode/mnode/impl/src/mndStb.c | 11 ++++++---- source/dnode/vnode/src/tq/tq.c | 14 +++--------- source/libs/stream/inc/streamInc.h | 1 - source/libs/stream/src/stream.c | 8 +++++-- source/libs/stream/src/streamMeta.c | 25 ++++++++++++++++------ source/libs/stream/src/streamQueue.c | 4 +++- source/libs/stream/src/streamTask.c | 4 ++-- source/util/src/terror.c | 1 + 13 files changed, 53 insertions(+), 31 deletions(-) diff --git a/examples/c/stream_demo.c b/examples/c/stream_demo.c index dd4fbc8d2d..2fcf4dd62c 100644 --- a/examples/c/stream_demo.c +++ b/examples/c/stream_demo.c @@ -98,10 +98,9 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = - taos_query(pConn, - "create stream stream1 trigger max_delay 10s into outstb as select _wstart, sum(k) from st1 partition " - "by tbname session(ts, 10s) "); + pRes = taos_query(pConn, + "create stream stream1 trigger max_delay 10s watermark 10s into outstb as select _wstart start, " + "count(k) from st1 partition by tbname interval(20s) "); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/common/tcommon.h b/include/common/tcommon.h index e04d9d5e86..dbe020f7ec 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -60,6 +60,7 @@ enum { STREAM_INPUT__DATA_RETRIEVE, STREAM_INPUT__GET_RES, STREAM_INPUT__CHECKPOINT, + STREAM_INPUT__DESTROY, }; typedef enum EStreamType { diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e6fcb021d5..484d0991f2 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -53,6 +53,7 @@ enum { TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE, TASK_SCHED_STATUS__FAILED, + TASK_SCHED_STATUS__DROPPING, }; enum { @@ -127,6 +128,10 @@ typedef struct { int8_t type; } SStreamCheckpoint; +typedef struct { + int8_t type; +} SStreamTaskDestroy; + typedef struct { int8_t type; SSDataBlock* pBlock; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index d7ec3697af..c3796fbadd 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -291,6 +291,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_STREAM_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F1) #define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2) #define TSDB_CODE_MND_STREAM_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03F3) +#define TSDB_CODE_MND_STREAM_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x03F4) // mnode-sma #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index a24b7ef459..3bfd7eb596 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -424,6 +424,8 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } mndAddTaskToTaskSet(taskSourceLevel, pTask); + pTask->triggerParam = 0; + // source pTask->taskLevel = TASK_LEVEL__SOURCE; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index dd2b595c29..59c6d65953 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2021,8 +2021,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, FOREACH(pNode, pNodeList) { SColumnNode *pCol = (SColumnNode *)pNode; - if (pCol->tableId != suid) { - mDebug("topic:%s, check colId:%d passed", pTopic->name, pCol->colId); + if (pCol->tableId == suid) { sdbRelease(pSdb, pTopic); nodesDestroyNode(pAst); return -1; @@ -2045,6 +2044,11 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) break; + if (pStream->smaId == 0 && pStream->targetStbUid == suid) { + sdbRelease(pSdb, pStream); + return -1; + } + SNode *pAst = NULL; if (nodesStringToNode(pStream->ast, &pAst) != 0) { ASSERT(0); @@ -2057,8 +2061,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, FOREACH(pNode, pNodeList) { SColumnNode *pCol = (SColumnNode *)pNode; - if (pCol->tableId != suid) { - mDebug("stream:%s, check colId:%d passed", pStream->name, pCol->colId); + if (pCol->tableId == suid) { sdbRelease(pSdb, pStream); nodesDestroyNode(pAst); return -1; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a98fea1988..c6bc8e6e59 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -628,8 +628,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { - int32_t code = 0; - if (pTask->taskLevel == TASK_LEVEL__AGG) { ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); } @@ -640,8 +638,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { pTask->outputQueue = streamQueueOpen(); if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) { - code = -1; - goto FAIL; + return -1; } pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; @@ -686,14 +683,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { streamSetupTrigger(pTask); - tqInfo("deploy stream task on vg %d, task id %d, child id %d", TD_VID(pTq->pVnode), pTask->taskId, + tqInfo("expand stream task on vg %d, task id %d, child id %d", TD_VID(pTq->pVnode), pTask->taskId, pTask->selfChildId); - -FAIL: - if (pTask->inputQueue) streamQueueClose(pTask->inputQueue); - if (pTask->outputQueue) streamQueueClose(pTask->outputQueue); - // TODO free executor - return code; + return 0; } int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 3776cb261f..6e30eeaa86 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -32,7 +32,6 @@ typedef struct { static SStreamGlobalEnv streamEnv; -int32_t streamExec(SStreamTask* pTask); int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch); int32_t streamDispatch(SStreamTask* pTask); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 6da7d4fd59..d6e87c2736 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -185,7 +185,9 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S tFreeStreamDispatchReq(pReq); if (exec) { - streamTryExec(pTask); + if (streamTryExec(pTask) < 0) { + return -1; + } if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamDispatch(pTask); @@ -221,7 +223,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp) { } int32_t streamProcessRunReq(SStreamTask* pTask) { - streamTryExec(pTask); + if (streamTryExec(pTask) < 0) { + return -1; + } if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamDispatch(pTask); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 64a9537e6c..f34f68ffc6 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -99,16 +99,19 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* goto FAIL; } - taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)); + if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + goto FAIL; + } if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, &pMeta->txn) < 0) { + taosHashRemove(pMeta->pTasks, &pTask->taskId, sizeof(int32_t)); ASSERT(0); - return -1; + goto FAIL; } return 0; FAIL: - if (pTask) taosMemoryFree(pTask); + if (pTask) tFreeSStreamTask(pTask); return -1; } @@ -158,11 +161,21 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask* pTask = *ppTask; taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING); + + if (tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), &pMeta->txn) < 0) { + /*return -1;*/ + } + + while (1) { + int8_t schedStatus = + atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__DROPPING); + if (schedStatus == TASK_SCHED_STATUS__INACTIVE) { + tFreeSStreamTask(pTask); + break; + } + } } - if (tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), &pMeta->txn) < 0) { - /*return -1;*/ - } return 0; } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 6819e5329f..45b78a8c6e 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -38,7 +38,9 @@ void streamQueueClose(SStreamQueue* queue) { if (qItem) { taosFreeQitem(qItem); } else { - return; + break; } } + taosFreeQall(queue->qall); + taosCloseQueue(queue->queue); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 638d39e5cc..0c35c1408e 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -152,8 +152,8 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { } void tFreeSStreamTask(SStreamTask* pTask) { - streamQueueClose(pTask->inputQueue); - streamQueueClose(pTask->outputQueue); + if (pTask->inputQueue) streamQueueClose(pTask->inputQueue); + if (pTask->outputQueue) streamQueueClose(pTask->outputQueue); if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg); if (pTask->exec.executor) qDestroyTask(pTask->exec.executor); taosMemoryFree(pTask); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 7b06967940..f6b62b5ea8 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -293,6 +293,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_ALREADY_EXIST, "Stream already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_NOT_EXIST, "Stream not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_OPTION, "Invalid stream option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_MUST_BE_DELETED, "Stream must be dropped first") // mnode-sma TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists") From e016c8f729c7d3677c66616257e2a15c856118d6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 17 Aug 2022 20:04:04 +0800 Subject: [PATCH 055/194] fix invalid packet --- source/libs/transport/inc/transComm.h | 6 ++++++ source/libs/transport/src/transCli.c | 1 + source/libs/transport/src/transComm.c | 8 ++++++-- source/libs/transport/src/transSvr.c | 16 ++++++++++------ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 1efb0bb316..bc1c6386f6 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -100,6 +100,10 @@ typedef void* queue[2]; #define TRANS_READ_TIMEOUT 3000 // read timeout (ms) #define TRANS_PACKET_LIMIT 1024 * 1024 * 512 +#define TRANS_MAGIC_NUM 0x5f375a86 + +#define TRANS_NOVALID_PACKET(src) ((src) != TRANS_MAGIC_NUM ? 1 : 0) + typedef SRpcMsg STransMsg; typedef SRpcCtx STransCtx; typedef SRpcCtxVal STransCtxVal; @@ -152,6 +156,7 @@ typedef struct { char hasEpSet : 2; // contain epset or not, 0(default): no epset, 1: contain epset char user[TSDB_UNI_LEN]; + uint32_t magicNum; STraceId traceId; uint64_t ahandle; // ahandle assigned by client uint32_t code; // del later @@ -204,6 +209,7 @@ typedef struct SConnBuffer { int cap; int left; int total; + int invalid; } SConnBuffer; typedef void (*AsyncCB)(uv_async_t* handle); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index add007e14d..8fdfcd5309 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -759,6 +759,7 @@ void cliSend(SCliConn* pConn) { pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); pHead->traceId = pMsg->info.traceId; + pHead->magicNum = htonl(TRANS_MAGIC_NUM); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index af49e5845b..3ba8186e9d 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -91,6 +91,7 @@ int transInitBuffer(SConnBuffer* buf) { buf->left = -1; buf->len = 0; buf->total = 0; + buf->invalid = 0; return 0; } int transDestroyBuffer(SConnBuffer* p) { @@ -108,6 +109,7 @@ int transClearBuffer(SConnBuffer* buf) { p->left = -1; p->len = 0; p->total = 0; + p->invalid = 0; return 0; } @@ -119,7 +121,7 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) { return -1; } int total = connBuf->total; - if (total >= HEADSIZE) { + if (total >= HEADSIZE && !p->invalid) { *buf = taosMemoryCalloc(1, total); memcpy(*buf, p->buf, total); transResetBuffer(connBuf); @@ -178,6 +180,7 @@ bool transReadComplete(SConnBuffer* connBuf) { memcpy((char*)&head, connBuf->buf, sizeof(head)); int32_t msgLen = (int32_t)htonl(head.msgLen); p->total = msgLen; + p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum)); } if (p->total >= p->len) { p->left = p->total - p->len; @@ -185,7 +188,8 @@ bool transReadComplete(SConnBuffer* connBuf) { p->left = 0; } } - return p->left == 0 ? true : false; + + return (p->left == 0 || p->invalid) ? true : false; } int transSetConnOption(uv_tcp_t* stream) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index a94eb01beb..be6eafba97 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -183,15 +183,14 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) { tDebug("%p timeout since no activity", conn); } -static void uvHandleReq(SSvrConn* pConn) { +static bool uvHandleReq(SSvrConn* pConn) { STrans* pTransInst = pConn->pTransInst; STransMsgHead* msg = NULL; int msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&msg); if (msgLen <= 0) { - tError("%s conn %p alread read complete packet", transLabel(pTransInst), pConn); - transUnrefSrvHandle(pConn); - return; + tError("%s conn %p read invalid packet", transLabel(pTransInst), pConn); + return false; } STransMsgHead* pHead = (STransMsgHead*)msg; @@ -207,7 +206,7 @@ static void uvHandleReq(SSvrConn* pConn) { // uv_queue_work(((SWorkThrd*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask); if (uvRecvReleaseReq(pConn, pHead)) { - return; + return true; } STransMsg transMsg; @@ -262,6 +261,7 @@ static void uvHandleReq(SSvrConn* pConn) { transReleaseExHandle(transGetRefMgt(), pConn->refId); (*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); + return true; } void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { @@ -275,7 +275,10 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { if (pBuf->len <= TRANS_PACKET_LIMIT) { while (transReadComplete(pBuf)) { tTrace("%s conn %p alread read complete packet", transLabel(pTransInst), conn); - uvHandleReq(conn); + if (uvHandleReq(conn) == false) { + destroyConn(conn, true); + break; + } } return; } else { @@ -374,6 +377,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { pHead->ahandle = (uint64_t)pMsg->info.ahandle; pHead->traceId = pMsg->info.traceId; pHead->hasEpSet = pMsg->info.hasEpSet; + pHead->magicNum = htonl(TRANS_MAGIC_NUM); if (pConn->status == ConnNormal) { pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType); From 32be8a71ee0c61f6fd52545fdc747c85ef23cf68 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 17 Aug 2022 20:14:06 +0800 Subject: [PATCH 056/194] enh: rsma batch process --- source/dnode/vnode/src/inc/sma.h | 29 ++- source/dnode/vnode/src/sma/smaCommit.c | 91 ++++---- source/dnode/vnode/src/sma/smaEnv.c | 18 +- source/dnode/vnode/src/sma/smaRollup.c | 302 +++++++++++++++---------- 4 files changed, 259 insertions(+), 181 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index c36207e495..26adc8d5e5 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -96,10 +96,10 @@ struct SRSmaStat { int8_t commitStat; // 0 not in committing, 1 in committing int8_t execStat; // 0 not in exec , 1 in exec SArray *aTaskFile; // qTaskFiles committed recently(for recovery/snapshot r/w) - SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; + SHashObj *infoHash; // key: suid, value: SRSmaInfo + SHashObj *fetchHash; // key: suid, value: L1 or L2 or L1|L2 }; - struct SSmaStat { union { STSmaStat tsmaStat; // time-range-wise sma @@ -108,13 +108,14 @@ struct SSmaStat { T_REF_DECLARE() }; -#define SMA_STAT_TSMA(s) (&(s)->tsmaStat) -#define SMA_STAT_RSMA(s) (&(s)->rsmaStat) -#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) -#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) -#define RSMA_COMMIT_STAT(r) (&(r)->commitStat) -#define RSMA_REF_ID(r) ((r)->refId) -#define RSMA_FS_LOCK(r) (&(r)->lock) +#define SMA_STAT_TSMA(s) (&(s)->tsmaStat) +#define SMA_STAT_RSMA(s) (&(s)->rsmaStat) +#define RSMA_INFO_HASH(r) ((r)->infoHash) +#define RSMA_FETCH_HASH(r) ((r)->fetchHash) +#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) +#define RSMA_COMMIT_STAT(r) (&(r)->commitStat) +#define RSMA_REF_ID(r) ((r)->refId) +#define RSMA_FS_LOCK(r) (&(r)->lock) struct SRSmaInfoItem { int8_t level; @@ -142,7 +143,7 @@ struct SRSmaInfo { #define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1) #define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1) #define RSMA_INFO_QTASK(r, i) ((r)->taskInfo[i]) -#define RSMA_INFO_IQTASK(r, i) ((r)->iTaskInfo[i]) +#define RSMA_INFO_IQTASK(r, i) ((r)->iTaskInfo[i]) #define RSMA_INFO_ITEM(r, i) (&(r)->items[i]) enum { @@ -167,6 +168,12 @@ enum { RSMA_RESTORE_SYNC = 2, }; +typedef enum { + RSMA_EXEC_OVERFLOW = 1, // triggered by queue buf overflow + RSMA_EXEC_TIMEOUT = 2, // triggered by timer + RSMA_EXEC_COMMIT = 3, // triggered by commit +} ERsmaExecType; + void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -240,7 +247,7 @@ static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); -int32_t tdRSmaProcessExecImpl(SSma *pSma, int8_t type); +int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type); int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdProcessRSmaRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 101fca3346..25777f90ab 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -121,7 +121,7 @@ static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); SRSmaStat *pRSmaStat = SMA_STAT_RSMA(pStat); // step 1: set rsma stat paused @@ -333,7 +333,34 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } } - // step 3: swap queue/qall and iQueue/iQal + /** + * @brief step 3: consume the SubmitReq in buffer + * 1) This is high cost task and should not put in asyncPreCommit originally. + * 2) But, if put in asyncCommit, would trigger taskInfo cloning frequently. + */ + nLoops = 0; + smaInfo("vgId:%d, start to wait for rsma qtask free, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); + + int8_t old; + while (1) { + old = atomic_val_compare_exchange_8(&pRSmaStat->execStat, 0, 1); + if (old == 0) break; + if (++nLoops > 1000) { + sched_yield(); + nLoops = 0; + smaDebug("vgId:%d, wait for rsma qtask free, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); + } + } + + smaInfo("vgId:%d, end to wait for rsma qtask free, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); + + if (tdRSmaProcessExecImpl(pSma, RSMA_EXEC_COMMIT) < 0) { + atomic_store_8(&pRSmaStat->execStat, 0); + return TSDB_CODE_FAILED; + } + + + // step 4: swap queue/qall and iQueue/iQall // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); @@ -351,11 +378,12 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } atomic_store_64(&pRSmaStat->qBufSize, 0); - + atomic_store_8(&pRSmaStat->execStat, 0); // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - // step 4: others + + // step 5: others pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; return TSDB_CODE_SUCCESS; @@ -375,36 +403,17 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); - // step 1: consume the SubmitReq in buffer - int32_t nLoops = 0; - smaDebug("vgId:%d start to wait for rsma qtask free, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - while (pRSmaStat->execStat == 1) { - taosMsleep(15); - if ((++nLoops & 63) == 0) { - smaWarn("vgId:%d 1s waited for rsma exec stat = 0, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - sched_yield(); - } - } - pRSmaStat->execStat = 1; - smaDebug("vgId:%d end to wait for rsma qtask free, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - if (tdRSmaProcessExecImpl(pSma, 1) < 0) { - pRSmaStat->execStat = 0; - return TSDB_CODE_FAILED; - } - - // step 2: perform persist task for qTaskInfo operator + // perform persist task for qTaskInfo operator if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) { - pRSmaStat->execStat = 0; return TSDB_CODE_FAILED; } - pRSmaStat->execStat = 0; return TSDB_CODE_SUCCESS; } /** - * @brief Migrate rsmaInfo from iRsmaInfo to rsmaInfo if rsmaInfoHash not empty. + * @brief Migrate rsmaInfo from iRsmaInfo to rsmaInfo if rsma infoHash not empty. * * @param pSma * @return int32_t @@ -424,13 +433,13 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { void *pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); while (pIter) { - tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); + tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; if (RSMA_INFO_IS_DEL(pRSmaInfo)) { int32_t refVal = T_REF_VAL_GET(pRSmaInfo); if (refVal == 0) { - if(!rsmaDeleted) { - if((rsmaDeleted = taosArrayInit(1, sizeof(tb_uid_t)))){ + if (!rsmaDeleted) { + if ((rsmaDeleted = taosArrayInit(1, sizeof(tb_uid_t)))) { taosArrayPush(rsmaDeleted, pSuid); } } @@ -461,22 +470,20 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter); } - if (taosArrayGetSize(rsmaDeleted) > 0) { - for (int32_t i = 0; i < taosArrayGetSize(rsmaDeleted); ++i) { - tb_uid_t *pSuid = taosArrayGet(rsmaDeleted, i); - void *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t)); - if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - tdFreeRSmaInfo(pSma, pRSmaInfo, true); - smaDebug( - "vgId:%d, rsma async post commit, free rsma info since already deleted and ref is 0 for " - "table:%" PRIi64, - SMA_VID(pSma), *pSuid); - } - taosHashRemove(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t)); + for (int32_t i = 0; i < taosArrayGetSize(rsmaDeleted); ++i) { + tb_uid_t *pSuid = taosArrayGet(rsmaDeleted, i); + void *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t)); + if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + tdFreeRSmaInfo(pSma, pRSmaInfo, true); + smaDebug( + "vgId:%d, rsma async post commit, free rsma info since already deleted and ref is 0 for " + "table:%" PRIi64, + SMA_VID(pSma), *pSuid); } - // remove suid in files - taosArrayDestroy(rsmaDeleted); + taosHashRemove(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t)); } + taosArrayDestroy(rsmaDeleted); + // TODO: remove suid in files? // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 73f8060559..f51aad22bd 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -171,7 +171,7 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { if (!pRSmaInfo) return 0; - + int ref = T_REF_INC(pRSmaInfo); smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); return 0; @@ -228,7 +228,12 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS RSMA_INFO_HASH(pRSmaStat) = taosHashInit( RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (!RSMA_INFO_HASH(pRSmaStat)) { - taosMemoryFreeClear(*pSmaStat); + return TSDB_CODE_FAILED; + } + + RSMA_FETCH_HASH(pRSmaStat) = taosHashInit( + RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + if (!RSMA_FETCH_HASH(pRSmaStat)) { return TSDB_CODE_FAILED; } } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { @@ -274,7 +279,10 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { } taosHashCleanup(RSMA_INFO_HASH(pStat)); - // step 3: wait all triggered fetch tasks finished + // step 3: destroy the rsma fetch hash + taosHashCleanup(RSMA_FETCH_HASH(pStat)); + + // step 4: wait all triggered fetch tasks finished int32_t nLoops = 0; while (1) { if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) { @@ -289,8 +297,8 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { nLoops = 0; } } - - // step 4: free pStat + + // step 5: free pStat taosMemoryFreeClear(pStat); } } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index de4b7dd808..1e2a8b35a4 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -17,7 +17,8 @@ #define RSMA_QTASKINFO_BUFSIZE (32768) #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid -#define RSMA_QTASKEXEC_BUFSIZ (1048576) +#define RSMA_QTASKEXEC_BUFSIZE (1048576) +#define RSMA_SUBMIT_BATCH_SIZE (1024) SSmaMgmt smaMgmt = { .inited = 0, @@ -35,9 +36,11 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUi static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx); static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo, - int8_t type, int8_t level); + ERsmaExecType type, int8_t level); static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid); static void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); +static void tdFreeRSmaSubmitItems(SArray *pItems); +static int32_t tdRSmaConsumeAndFetch(SSma *pSma, int64_t suid, int8_t level, SArray *pSubmitArr); static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid); static void tdRSmaFetchTrigger(void *param, void *tmrId); @@ -559,6 +562,14 @@ void *tdUidStoreFree(STbUidStore *pStore) { return NULL; } +/** + * @brief The SubmitReq for rsma L2/L3 is inserted by tsdbInsertData method directly while not by WriteQ, as the queue + * would be freed when close Vnode, thus lock should be used if with race condition. + * @param pTsdb + * @param version + * @param pReq + * @return int32_t + */ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { if (!pReq) { terrno = TSDB_CODE_INVALID_PTR; @@ -566,7 +577,7 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { } SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - + // TODO: spin lock for race conditiond if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { return TSDB_CODE_FAILED; } @@ -696,7 +707,7 @@ _err: } /** - * @brief Copy msg to rsmaQueueBuffer + * @brief Copy msg to rsmaQueueBuffer for batch process * * @param pSma * @param pMsg @@ -722,17 +733,17 @@ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inpu int64_t bufSize = atomic_add_fetch_64(&pRSmaStat->qBufSize, pReq->header.contLen); // smoothing consume - int32_t n = bufSize / RSMA_QTASKEXEC_BUFSIZ; + int32_t n = bufSize / RSMA_QTASKEXEC_BUFSIZE; if (n > 1) { if (n > 10) { n = 10; } taosMsleep(n << 4); if (n > 2) { - smaWarn("vgId:%d pInfo->queue itemSize:%d, memSize:%" PRIi64 ", sleep %d ms", SMA_VID(pSma), + smaWarn("vgId:%d, pInfo->queue itemSize:%d, memSize:%" PRIi64 ", sleep %d ms", SMA_VID(pSma), taosQueueItemSize(pInfo->queue), taosQueueMemorySize(pInfo->queue), n << 4); } else { - smaDebug("vgId:%d pInfo->queue itemSize:%d, memSize:%" PRIi64 ", sleep %d ms", SMA_VID(pSma), + smaDebug("vgId:%d, pInfo->queue itemSize:%d, memSize:%" PRIi64 ", sleep %d ms", SMA_VID(pSma), taosQueueItemSize(pInfo->queue), taosQueueMemorySize(pInfo->queue), n << 4); } } @@ -751,7 +762,7 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { if (pBlock == NULL) break; tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { - smaDebug("vgId:%d numOfRows:%d, suid:%" PRIi64 ", uid:%" PRIi64 ", version:%" PRIi64 ", ts:%" PRIi64, + smaDebug("vgId:%d, numOfRows:%d, suid:%" PRIi64 ", uid:%" PRIi64 ", version:%" PRIi64 ", ts:%" PRIi64, SMA_VID(pSma), msgIter.numOfRows, msgIter.suid, msgIter.uid, pReq->version, row->ts); } } @@ -771,10 +782,10 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { * @return int32_t */ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo, - int8_t type, int8_t level) { + ERsmaExecType type, int8_t level) { int32_t idx = level - 1; - void *qTaskInfo = (type == 0) ? RSMA_INFO_QTASK(pInfo, idx) : RSMA_INFO_IQTASK(pInfo, idx); + void *qTaskInfo = (type == RSMA_EXEC_COMMIT) ? RSMA_INFO_IQTASK(pInfo, idx) : RSMA_INFO_QTASK(pInfo, idx); if (!qTaskInfo) { smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, pInfo->suid); @@ -791,7 +802,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, #if 0 for (int32_t i = 0; i < msgSize; ++i) { SSubmitReq *pReq = *(SSubmitReq **)((char *)pMsg + i * sizeof(void *)); - smaDebug("vgId:%d [%d][%d] version %" PRIi64, SMA_VID(pSma), msgSize, i, pReq->version); + smaDebug("vgId:%d, [%d][%d] version %" PRIi64, SMA_VID(pSma), msgSize, i, pReq->version); tdRsmaPrintSubmitReq(pSma, pReq); } #endif @@ -802,11 +813,6 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); tdRSmaFetchAndSubmitResult(pSma, qTaskInfo, pItem, pInfo->pTSchema, pInfo->suid); - atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); - - if (smaMgmt.tmrHandle) { - taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId); - } return TSDB_CODE_SUCCESS; } @@ -854,13 +860,7 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { } taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - if (RSMA_COMMIT_STAT(pStat) == 0) { // return NULL if not in committing stat - return NULL; - } - - // unlock - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - return pRSmaInfo; + return NULL; } static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { @@ -890,6 +890,16 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_FAILED; } + if (smaMgmt.tmrHandle) { + SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, 0); + if (pItem->level > 0) { + atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); + } + pItem = RSMA_INFO_ITEM(pRSmaInfo, 1); + if (pItem->level > 0) { + atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); + } + } } else { ASSERT(0); } @@ -898,51 +908,23 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp return TSDB_CODE_SUCCESS; } -#if 0 -/** - * @brief sync mode - * - * @param pSma - * @param pMsg - * @param inputType - * @param suid - * @return int32_t - */ -static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { - SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid); - if (!pRSmaInfo) { - smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); - return TSDB_CODE_SUCCESS; - } - - if (inputType == STREAM_INPUT__DATA_SUBMIT) { - tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo, suid, TSDB_RETENTION_L1); - tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo, suid, TSDB_RETENTION_L2); - } - - tdReleaseRSmaInfo(pSma, pRSmaInfo); - return TSDB_CODE_SUCCESS; -} -#endif - static int32_t tdRSmaExecCheck(SSma *pSma) { - SRSmaStat *pRsmaStat = SMA_RSMA_STAT(pSma); - int64_t bufSize = atomic_load_64(&pRsmaStat->qBufSize); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); + int64_t bufSize = atomic_load_64(&pRSmaStat->qBufSize); - if ((pRsmaStat->execStat == 1) || (bufSize < RSMA_QTASKEXEC_BUFSIZ)) { - if (bufSize > RSMA_QTASKEXEC_BUFSIZ) { - smaDebug("vgId:%d bufSize is %d but has no chance to exec as qTaskInfo occupied by another task", SMA_VID(pSma), - bufSize); - } else { - smaDebug("vgId:%d bufSize is %d but has no chance to exec as less than %d", SMA_VID(pSma), bufSize, - RSMA_QTASKEXEC_BUFSIZ); - } + if (bufSize < RSMA_QTASKEXEC_BUFSIZE) { + smaDebug("vgId:%d, bufSize is %d but has no chance to exec as less than %d", SMA_VID(pSma), bufSize, + RSMA_QTASKEXEC_BUFSIZE); return TSDB_CODE_SUCCESS; } - smaDebug("vgId:%d bufSize is %d and has chance to exec as qTaskInfo is free now", SMA_VID(pSma), bufSize); + if (atomic_val_compare_exchange_8(&pRSmaStat->execStat, 0, 1) == 1) { + smaDebug("vgId:%d, bufSize is %d but has no chance to exec as qTaskInfo occupied by another task", SMA_VID(pSma), + bufSize); + return TSDB_CODE_SUCCESS; + } - pRsmaStat->execStat = 1; + smaDebug("vgId:%d, bufSize is %d and has chance to exec as qTaskInfo is free now", SMA_VID(pSma), bufSize); SRSmaExecMsg fetchMsg; int32_t contLen = sizeof(SMsgHead); @@ -967,7 +949,7 @@ static int32_t tdRSmaExecCheck(SSma *pSma) { return TSDB_CODE_SUCCESS; _err: - pRsmaStat->execStat = 0; + atomic_store_8(&pRSmaStat->execStat, 0); return TSDB_CODE_FAILED; } @@ -1592,7 +1574,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { } _end: - // taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId); + taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId); tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); } @@ -1656,13 +1638,11 @@ _err: * @return int32_t */ int32_t smaProcessFetch(SSma *pSma, void *pMsg) { - SRpcMsg *pRpcMsg = (SRpcMsg *)pMsg; - SRSmaFetchMsg req = {0}; - SDecoder decoder = {0}; - void *pBuf = NULL; - SRSmaInfo *pInfo = NULL; - SRSmaInfoItem *pItem = NULL; - + SRpcMsg *pRpcMsg = (SRpcMsg *)pMsg; + SRSmaFetchMsg req = {0}; + SDecoder decoder = {0}; + void *pBuf = NULL; + SRSmaStat *pRSmaStat = NULL; if (!pRpcMsg || pRpcMsg->contLen < sizeof(SMsgHead)) { terrno = TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP; goto _err; @@ -1676,38 +1656,33 @@ int32_t smaProcessFetch(SSma *pSma, void *pMsg) { goto _err; } -#if 0 - pInfo = tdAcquireRSmaInfoBySuid(pSma, req.suid); - if (!pInfo) { - if (terrno == TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_RSMA_EMPTY_INFO; + pRSmaStat = SMA_RSMA_STAT(pSma); + + if (atomic_val_compare_exchange_8(&pRSmaStat->execStat, 0, 1) == 0) { + SArray *pSubmitArr = NULL; + if (!(pSubmitArr = taosArrayInit(RSMA_SUBMIT_BATCH_SIZE, POINTER_BYTES))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + atomic_store_8(&pRSmaStat->execStat, 0); + goto _err; } - smaWarn("vgId:%d, failed to process rsma fetch msg for suid:%" PRIi64 " level:%" PRIi8 " since %s", SMA_VID(pSma), - req.suid, req.level, terrstr()); - goto _err; + tdRSmaConsumeAndFetch(pSma, req.suid, req.level, pSubmitArr); + atomic_store_8(&pRSmaStat->execStat, 0); + taosArrayDestroy(pSubmitArr); + } else { + int8_t level = req.level; + int8_t *val = taosHashGet(RSMA_FETCH_HASH(pRSmaStat), &req.suid, sizeof(req.suid)); + if (val) { + level |= (*val); + } + ASSERT(level >= 1 && level <= 3); + taosHashPut(RSMA_FETCH_HASH(pRSmaStat), &req.suid, sizeof(req.suid), &level, sizeof(level)); } - pItem = RSMA_INFO_ITEM(pInfo, req.level - 1); - - SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pInfo, req.level - 1); - if ((terrno = qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) { - goto _err; - } - if (tdRSmaFetchAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid) < 0) { - goto _err; - } - - tdCleanupStreamInputDataBlock(taskInfo); - - tdReleaseRSmaInfo(pSma, pInfo); -#endif tDecoderClear(&decoder); smaDebug("vgId:%d, success to process rsma fetch msg for suid:%" PRIi64 " level:%" PRIi8, SMA_VID(pSma), req.suid, req.level); return TSDB_CODE_SUCCESS; _err: - // tdReleaseRSmaInfo(pSma, pInfo); tDecoderClear(&decoder); smaError("vgId:%d, failed to process rsma fetch msg since %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; @@ -1719,28 +1694,101 @@ static void tdFreeRSmaSubmitItems(SArray *pItems) { } } +static int32_t tdRSmaConsumeAndFetch(SSma *pSma, int64_t suid, int8_t level, SArray *pSubmitArr) { + SRSmaInfo *pInfo = tdAcquireRSmaInfoBySuid(pSma, suid); + if (!pInfo) { + return TSDB_CODE_SUCCESS; + } + + // step 1: consume submit req + int64_t qMemSize = 0; + if ((qMemSize = taosQueueMemorySize(pInfo->queue) > 0)) { + taosReadAllQitems(pInfo->queue, pInfo->qall); // queue has mutex lock + + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); + atomic_fetch_sub_64(&pRSmaStat->qBufSize, qMemSize); + + taosArrayClear(pSubmitArr); + + while (1) { + void *msg = NULL; + taosGetQitem(pInfo->qall, (void **)&msg); + if (msg) { + if (taosArrayPush(pSubmitArr, &msg) < 0) { + tdFreeRSmaSubmitItems(pSubmitArr); + goto _err; + } + } else { + break; + } + } + + int32_t size = taosArrayGetSize(pSubmitArr); + if (size > 0) { + for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { + if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, RSMA_EXEC_TIMEOUT, i) < + 0) { + tdFreeRSmaSubmitItems(pSubmitArr); + goto _err; + } + } + + tdFreeRSmaSubmitItems(pSubmitArr); + } + } + + // step 2: fetch rsma result + SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; + for (int8_t i = 1; i <= TSDB_RETENTION_L2; ++i) { + if (level & i) { + qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pInfo, i - 1); + if (!taskInfo) { + continue; + } + if ((terrno = qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) { + goto _err; + } + SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, i - 1); + if (tdRSmaFetchAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, suid) < 0) { + tdCleanupStreamInputDataBlock(taskInfo); + goto _err; + } + + tdCleanupStreamInputDataBlock(taskInfo); + } + } + +_end: + tdReleaseRSmaInfo(pSma, pInfo); + return TSDB_CODE_SUCCESS; +_err: + tdReleaseRSmaInfo(pSma, pInfo); + return TSDB_CODE_FAILED; +} + /** * @brief * * @param pSma - * @param type 0 triggered when buffer overflow, 1 triggered by commit + * @param type * @return int32_t */ -int32_t tdRSmaProcessExecImpl(SSma *pSma, int8_t type) { +int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SHashObj *infoHash = NULL; SArray *pSubmitQArr = NULL; SArray *pSubmitArr = NULL; + bool isFetchAll = false; if (!pRSmaStat || !(infoHash = RSMA_INFO_HASH(pRSmaStat))) { terrno = TSDB_CODE_RSMA_INVALID_STAT; goto _err; } - if (type == 0) { + if (type == RSMA_EXEC_OVERFLOW) { taosRLockLatch(SMA_ENV_LOCK(pEnv)); - if (atomic_load_64(&pRSmaStat->qBufSize) < RSMA_QTASKEXEC_BUFSIZ) { + if (atomic_load_64(&pRSmaStat->qBufSize) < RSMA_QTASKEXEC_BUFSIZE) { taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return TSDB_CODE_SUCCESS; } @@ -1752,19 +1800,19 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, int8_t type) { goto _err; } - if (!(pSubmitArr = taosArrayInit(1024, POINTER_BYTES))) { + if (!(pSubmitArr = taosArrayInit(RSMA_SUBMIT_BATCH_SIZE, POINTER_BYTES))) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + // step 1: rsma exec - consume data in buffer queue for all suids SRSmaExecQItem qItem = {0}; - taosWLockLatch(SMA_ENV_LOCK(pEnv)); - void *pIter = taosHashIterate(infoHash, NULL); - if (type == 0) { + void *pIter = taosHashIterate(infoHash, NULL); // infoHash has r/w lock + if (type == RSMA_EXEC_OVERFLOW) { while (pIter) { SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; if (taosQueueItemSize(pInfo->queue)) { - taosReadAllQitems(pInfo->queue, pInfo->qall); + taosReadAllQitems(pInfo->queue, pInfo->qall); // queue has mutex lock qItem.qall = &pInfo->qall; qItem.pRSmaInfo = pIter; taosArrayPush(pSubmitQArr, &qItem); @@ -1772,7 +1820,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, int8_t type) { ASSERT(taosQueueItemSize(pInfo->queue) == 0); pIter = taosHashIterate(infoHash, pIter); } - } else if (type == 1) { + } else if (type == RSMA_EXEC_COMMIT) { while (pIter) { SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; if (taosQueueItemSize(pInfo->iQueue)) { @@ -1788,7 +1836,6 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, int8_t type) { ASSERT(0); } atomic_store_64(&pRSmaStat->qBufSize, 0); - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); int32_t qSize = taosArrayGetSize(pSubmitQArr); for (int32_t i = 0; i < qSize; ++i) { @@ -1808,22 +1855,31 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, int8_t type) { int32_t size = taosArrayGetSize(pSubmitArr); if (size > 0) { - if (type == 0 || type == 1) { - SRSmaInfo *pInfo = *(SRSmaInfo **)pItem->pRSmaInfo; - for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { - if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, type, i) < 0) { - tdFreeRSmaSubmitItems(pSubmitArr); - goto _err; - } + SRSmaInfo *pInfo = *(SRSmaInfo **)pItem->pRSmaInfo; + for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { + if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, type, i) < 0) { + tdFreeRSmaSubmitItems(pSubmitArr); + goto _err; } - } else { - ASSERT(0); } tdFreeRSmaSubmitItems(pSubmitArr); taosArrayClear(pSubmitArr); } } + // step 2: rsma fetch - consume data in buffer queue for suids triggered by timer + if (taosHashGetSize(RSMA_FETCH_HASH(pRSmaStat)) <= 0) { + goto _end; + } + pIter = taosHashIterate(RSMA_FETCH_HASH(pRSmaStat), NULL); + if (pIter) { + tdRSmaConsumeAndFetch(pSma, *(int64_t *)taosHashGetKey(pIter, NULL), *(int8_t *)pIter, pSubmitArr); + while ((pIter = taosHashIterate(RSMA_FETCH_HASH(pRSmaStat), pIter))) { + tdRSmaConsumeAndFetch(pSma, *(int64_t *)taosHashGetKey(pIter, NULL), *(int8_t *)pIter, pSubmitArr); + } + } + +_end: taosArrayDestroy(pSubmitArr); taosArrayDestroy(pSubmitQArr); return TSDB_CODE_SUCCESS; @@ -1842,23 +1898,23 @@ _err: */ int32_t smaProcessExec(SSma *pSma, void *pMsg) { SRpcMsg *pRpcMsg = (SRpcMsg *)pMsg; - SRSmaStat *pRsmaStat = SMA_RSMA_STAT(pSma); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); if (!pRpcMsg || pRpcMsg->contLen < sizeof(SMsgHead)) { terrno = TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP; goto _err; } - smaDebug("vgId:%d, begin to process rsma exec msg by thread:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - if (tdRSmaProcessExecImpl(pSma, 0) < 0) { + smaDebug("vgId:%d, begin to process rsma exec msg by TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); + if (tdRSmaProcessExecImpl(pSma, RSMA_EXEC_OVERFLOW) < 0) { goto _err; } - pRsmaStat->execStat = 0; - smaDebug("vgId:%d, success to process rsma exec msg by thead:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); + atomic_store_8(&pRSmaStat->execStat, 0); + smaDebug("vgId:%d, success to process rsma exec msg by TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); return TSDB_CODE_SUCCESS; _err: - pRsmaStat->execStat = 0; - smaError("vgId:%d, failed to process rsma fetch msg by thread:%p since %s", SMA_VID(pSma), - (void *)taosGetSelfPthreadId(), terrstr()); + atomic_store_8(&pRSmaStat->execStat, 0); + smaError("vgId:%d, failed to process rsma exec msg by TID:%p since %s", SMA_VID(pSma), (void *)taosGetSelfPthreadId(), + terrstr()); return TSDB_CODE_FAILED; } From ea891dc79353ee862509bf26ae0182e7d879b69b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 17 Aug 2022 20:30:42 +0800 Subject: [PATCH 057/194] other: code optimization --- source/dnode/vnode/src/vnd/vnodeCommit.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 2f5169a0ec..ab618fe430 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -222,6 +222,13 @@ int vnodeCommit(SVnode *pVnode) { pVnode->state.commitTerm = pVnode->state.applyTerm; + // preCommit + // smaSyncPreCommit(pVnode->pSma); + smaAsyncPreCommit(pVnode->pSma); + + vnodeBufPoolUnRef(pVnode->inUse); + pVnode->inUse = NULL; + // save info info.config = pVnode->config; info.state.committed = pVnode->state.applied; @@ -234,10 +241,6 @@ int vnodeCommit(SVnode *pVnode) { } walBeginSnapshot(pVnode->pWal, pVnode->state.applied); - // preCommit - // smaSyncPreCommit(pVnode->pSma); - smaAsyncPreCommit(pVnode->pSma); - // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { ASSERT(0); @@ -245,7 +248,7 @@ int vnodeCommit(SVnode *pVnode) { } if (VND_IS_RSMA(pVnode)) { - smaAsyncCommit(pVnode->pSma); // would write L2/L3 data into BufPool + smaAsyncCommit(pVnode->pSma); if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { ASSERT(0); From d65b75514a130983f0cb035f91516e72cc167a88 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 17 Aug 2022 20:38:34 +0800 Subject: [PATCH 058/194] fix: remove obsoleted codes --- source/dnode/vnode/src/vnd/vnodeCommit.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index ab618fe430..fc09eaac44 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -268,9 +268,6 @@ int vnodeCommit(SVnode *pVnode) { return -1; } } - - vnodeBufPoolUnRef(pVnode->inUse); - pVnode->inUse = NULL; if (tqCommit(pVnode->pTq) < 0) { ASSERT(0); From c6f60ba3d78e92ef494c5d1129e998a40f3f4d2d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 17 Aug 2022 20:49:59 +0800 Subject: [PATCH 059/194] fix invalid packet --- source/libs/transport/inc/transComm.h | 8 +++ source/libs/transport/src/transCli.c | 66 +++++++++++------- source/libs/transport/src/transComm.c | 19 +++-- source/libs/transport/src/transSvr.c | 99 +++++++++++++++------------ 4 files changed, 119 insertions(+), 73 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 04b58da570..117455f722 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -99,6 +99,12 @@ typedef void* queue[2]; #define TRANS_CONN_TIMEOUT 3 // connect timeout (s) #define TRANS_READ_TIMEOUT 3000 // read timeout (ms) +#define TRANS_PACKET_LIMIT 1024 * 1024 * 512 + +#define TRANS_MAGIC_NUM 0x5f375a86 + +#define TRANS_NOVALID_PACKET(src) ((src) != TRANS_MAGIC_NUM ? 1 : 0) + typedef SRpcMsg STransMsg; typedef SRpcCtx STransCtx; typedef SRpcCtxVal STransCtxVal; @@ -151,6 +157,7 @@ typedef struct { char hasEpSet : 2; // contain epset or not, 0(default): no epset, 1: contain epset char user[TSDB_UNI_LEN]; + uint32_t magicNum; STraceId traceId; uint64_t ahandle; // ahandle assigned by client uint32_t code; // del later @@ -203,6 +210,7 @@ typedef struct SConnBuffer { int cap; int left; int total; + int invalid; } SConnBuffer; typedef void (*AsyncCB)(uv_async_t* handle); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 9eea43be23..5428b8acf6 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -127,6 +127,8 @@ static void cliAsyncCb(uv_async_t* handle); static void cliIdleCb(uv_idle_t* handle); static void cliPrepareCb(uv_prepare_t* handle); +static bool cliRecvReleaseReq(SCliConn* conn, STransMsgHead* pHead); + static int32_t allocConnRef(SCliConn* conn, bool update); static int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg); @@ -211,28 +213,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { #define CONN_PERSIST_TIME(para) ((para) <= 90000 ? 90000 : (para)) #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) #define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label) -#define CONN_SHOULD_RELEASE(conn, head) \ - do { \ - if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ - uint64_t ahandle = head->ahandle; \ - CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ - transClearBuffer(&conn->readBuf); \ - transFreeMsg(transContFromHead((char*)head)); \ - if (transQueueSize(&conn->cliMsgs) > 0 && ahandle == 0) { \ - SCliMsg* cliMsg = transQueueGet(&conn->cliMsgs, 0); \ - if (cliMsg->type == Release) return; \ - } \ - tDebug("%s conn %p receive release request, refId:%" PRId64 "", CONN_GET_INST_LABEL(conn), conn, conn->refId); \ - if (T_REF_VAL_GET(conn) > 1) { \ - transUnrefCliHandle(conn); \ - } \ - destroyCmsg(pMsg); \ - cliReleaseUnfinishedMsg(conn); \ - transQueueClear(&conn->cliMsgs); \ - addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ - return; \ - } \ - } while (0) #define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \ do { \ @@ -346,10 +326,17 @@ void cliHandleResp(SCliConn* conn) { } STransMsgHead* pHead = NULL; - transDumpFromBuffer(&conn->readBuf, (char**)&pHead); + if (transDumpFromBuffer(&conn->readBuf, (char**)&pHead) <= 0) { + tDebug("%s conn %p recv invalid packet ", CONN_GET_INST_LABEL(conn), conn); + return; + } pHead->code = htonl(pHead->code); pHead->msgLen = htonl(pHead->msgLen); + if (cliRecvReleaseReq(conn, pHead)) { + return; + } + STransMsg transMsg = {0}; transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.pCont = transContFromHead((char*)pHead); @@ -361,7 +348,6 @@ void cliHandleResp(SCliConn* conn) { SCliMsg* pMsg = NULL; STransConnCtx* pCtx = NULL; - CONN_SHOULD_RELEASE(conn, pHead); if (CONN_NO_PERSIST_BY_APP(conn)) { pMsg = transQueuePop(&conn->cliMsgs); @@ -625,7 +611,12 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { pBuf->len += nread; while (transReadComplete(pBuf)) { tTrace("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn); - cliHandleResp(conn); + if (pBuf->invalid) { + cliHandleExcept(conn); + break; + } else { + cliHandleResp(conn); + } } return; } @@ -786,6 +777,7 @@ void cliSend(SCliConn* pConn) { pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); pHead->traceId = pMsg->info.traceId; + pHead->magicNum = htonl(TRANS_MAGIC_NUM); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); @@ -1053,6 +1045,30 @@ static void cliPrepareCb(uv_prepare_t* handle) { if (thrd->stopMsg != NULL) cliHandleQuit(thrd->stopMsg, thrd); } +bool cliRecvReleaseReq(SCliConn* conn, STransMsgHead* pHead) { + if (pHead->release == 1 && (pHead->msgLen) == sizeof(*pHead)) { + uint64_t ahandle = pHead->ahandle; + SCliMsg* pMsg = NULL; + CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); + transClearBuffer(&conn->readBuf); + transFreeMsg(transContFromHead((char*)pHead)); + if (transQueueSize(&conn->cliMsgs) > 0 && ahandle == 0) { + SCliMsg* cliMsg = transQueueGet(&conn->cliMsgs, 0); + if (cliMsg->type == Release) return true; + } + tDebug("%s conn %p receive release request, refId:%" PRId64 "", CONN_GET_INST_LABEL(conn), conn, conn->refId); + if (T_REF_VAL_GET(conn) > 1) { + transUnrefCliHandle(conn); + } + destroyCmsg(pMsg); + cliReleaseUnfinishedMsg(conn); + transQueueClear(&conn->cliMsgs); + addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); + return true; + } + return false; +} + static void* cliWorkThread(void* arg) { SCliThrd* pThrd = (SCliThrd*)arg; pThrd->pid = taosGetSelfPthreadId(); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index b568163e23..c50d0d3e5c 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -91,6 +91,7 @@ int transInitBuffer(SConnBuffer* buf) { buf->left = -1; buf->len = 0; buf->total = 0; + buf->invalid = 0; return 0; } int transDestroyBuffer(SConnBuffer* p) { @@ -108,19 +109,24 @@ int transClearBuffer(SConnBuffer* buf) { p->left = -1; p->len = 0; p->total = 0; + p->invalid = 0; return 0; } int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) { - SConnBuffer* p = connBuf; + static const int HEADSIZE = sizeof(STransMsgHead); + SConnBuffer* p = connBuf; if (p->left != 0) { return -1; } int total = connBuf->total; - *buf = taosMemoryCalloc(1, total); - memcpy(*buf, p->buf, total); - - transResetBuffer(connBuf); + if (total >= HEADSIZE && !p->invalid) { + *buf = taosMemoryCalloc(1, total); + memcpy(*buf, p->buf, total); + transResetBuffer(connBuf); + } else { + total = -1; + } return total; } @@ -173,6 +179,7 @@ bool transReadComplete(SConnBuffer* connBuf) { memcpy((char*)&head, connBuf->buf, sizeof(head)); int32_t msgLen = (int32_t)htonl(head.msgLen); p->total = msgLen; + p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum)); } if (p->total >= p->len) { p->left = p->total - p->len; @@ -180,7 +187,7 @@ bool transReadComplete(SConnBuffer* connBuf) { p->left = 0; } } - return p->left == 0 ? true : false; + return (p->left == 0 || p->invalid) ? true : false; } int transSetConnOption(uv_tcp_t* stream) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 4d35e346b1..f8f55ab1d8 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -114,6 +114,8 @@ static void uvAcceptAsyncCb(uv_async_t* handle); static void uvShutDownCb(uv_shutdown_t* req, int status); static void uvPrepareCb(uv_prepare_t* handle); +static bool uvRecvReleaseReq(SSvrConn* conn, STransMsgHead* pHead); + /* * time-consuming task throwed into BG work thread */ @@ -154,37 +156,6 @@ static void* transAcceptThread(void* arg); static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static bool addHandleToAcceptloop(void* arg); -#define CONN_SHOULD_RELEASE(conn, head) \ - do { \ - if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ - reallocConnRef(conn); \ - tTrace("conn %p received release request", conn); \ - \ - STraceId traceId = head->traceId; \ - conn->status = ConnRelease; \ - transClearBuffer(&conn->readBuf); \ - transFreeMsg(transContFromHead((char*)head)); \ - \ - STransMsg tmsg = { \ - .code = 0, .info.handle = (void*)conn, .info.traceId = traceId, .info.ahandle = (void*)0x9527}; \ - SSvrMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSvrMsg)); \ - srvMsg->msg = tmsg; \ - srvMsg->type = Release; \ - srvMsg->pConn = conn; \ - if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ - return; \ - } \ - if (conn->regArg.init) { \ - tTrace("conn %p release, notify server app", conn); \ - STrans* pTransInst = conn->pTransInst; \ - (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); \ - memset(&conn->regArg, 0, sizeof(conn->regArg)); \ - } \ - uvStartSendRespInternal(srvMsg); \ - return; \ - } \ - } while (0) - #define SRV_RELEASE_UV(loop) \ do { \ uv_walk(loop, uvWalkCb, NULL); \ @@ -212,17 +183,25 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) { tDebug("%p timeout since no activity", conn); } -static void uvHandleReq(SSvrConn* pConn) { - STransMsgHead* msg = NULL; - int msgLen = 0; +static bool uvHandleReq(SSvrConn* pConn) { + STrans* pTransInst = pConn->pTransInst; - msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&msg); + STransMsgHead* msg = NULL; + int msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&msg); + if (msgLen <= 0) { + tError("%s conn %p read invalid packet", transLabel(pTransInst), pConn); + return false; + } STransMsgHead* pHead = (STransMsgHead*)msg; pHead->code = htonl(pHead->code); pHead->msgLen = htonl(pHead->msgLen); memcpy(pConn->user, pHead->user, strlen(pHead->user)); + if (uvRecvReleaseReq(pConn, pHead)) { + return true; + } + // TODO(dengyihao): time-consuming task throwed into BG Thread // uv_work_t* wreq = taosMemoryMalloc(sizeof(uv_work_t)); // wreq->data = pConn; @@ -230,8 +209,6 @@ static void uvHandleReq(SSvrConn* pConn) { // transRefSrvHandle(pConn); // uv_queue_work(((SWorkThrd*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask); - CONN_SHOULD_RELEASE(pConn, pHead); - STransMsg transMsg; memset(&transMsg, 0, sizeof(transMsg)); transMsg.contLen = transContLenFromMsg(pHead->msgLen); @@ -247,7 +224,6 @@ static void uvHandleReq(SSvrConn* pConn) { tDebug("conn %p acquired by server app", pConn); } } - STrans* pTransInst = pConn->pTransInst; STraceId* trace = &pHead->traceId; if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); @@ -285,6 +261,7 @@ static void uvHandleReq(SSvrConn* pConn) { transReleaseExHandle(transGetRefMgt(), pConn->refId); (*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); + return true; } void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { @@ -295,11 +272,18 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { if (nread > 0) { pBuf->len += nread; tTrace("%s conn %p total read:%d, current read:%d", transLabel(pTransInst), conn, pBuf->len, (int)nread); - while (transReadComplete(pBuf)) { - tTrace("%s conn %p alread read complete packet", transLabel(pTransInst), conn); - uvHandleReq(conn); + if (pBuf->len <= TRANS_PACKET_LIMIT) { + while (transReadComplete(pBuf)) { + tTrace("%s conn %p alread read complete packet", transLabel(pTransInst), conn); + if (pBuf->invalid) { + destroyConn(conn, true); + break; + } else { + if (false == uvHandleReq(conn)) break; + } + } + return; } - return; } if (nread == 0) { return; @@ -391,6 +375,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { pHead->ahandle = (uint64_t)pMsg->info.ahandle; pHead->traceId = pMsg->info.traceId; pHead->hasEpSet = pMsg->info.hasEpSet; + pHead->magicNum = htonl(TRANS_MAGIC_NUM); if (pConn->status == ConnNormal) { pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType); @@ -591,6 +576,36 @@ static void uvPrepareCb(uv_prepare_t* handle) { } } +static bool uvRecvReleaseReq(SSvrConn* pConn, STransMsgHead* pHead) { + if ((pHead)->release == 1 && (pHead->msgLen) == sizeof(*pHead)) { + reallocConnRef(pConn); + tTrace("conn %p received release request", pConn); + + STraceId traceId = pHead->traceId; + pConn->status = ConnRelease; + transClearBuffer(&pConn->readBuf); + transFreeMsg(transContFromHead((char*)pHead)); + + STransMsg tmsg = {.code = 0, .info.handle = (void*)pConn, .info.traceId = traceId, .info.ahandle = (void*)0x9527}; + SSvrMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSvrMsg)); + srvMsg->msg = tmsg; + srvMsg->type = Release; + srvMsg->pConn = pConn; + if (!transQueuePush(&pConn->srvMsgs, srvMsg)) { + return true; + } + if (pConn->regArg.init) { + tTrace("conn %p release, notify server app", pConn); + STrans* pTransInst = pConn->pTransInst; + (*pTransInst->cfp)(pTransInst->parent, &(pConn->regArg.msg), NULL); + memset(&pConn->regArg, 0, sizeof(pConn->regArg)); + } + uvStartSendResp(srvMsg); + return true; + } + return false; +} + static void uvWorkDoTask(uv_work_t* req) { // doing time-consumeing task // only auth conn currently, add more func later From 66f6dcddb85206630df1b1d339ee7915ec7269e3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 17 Aug 2022 23:02:22 +0800 Subject: [PATCH 060/194] fix invalid packet --- source/libs/transport/src/transSvr.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index f8f55ab1d8..f84b87a43d 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -276,6 +276,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { while (transReadComplete(pBuf)) { tTrace("%s conn %p alread read complete packet", transLabel(pTransInst), conn); if (pBuf->invalid) { + tTrace("%s conn %p alread read invalid packet", transLabel(pTransInst), conn); destroyConn(conn, true); break; } else { @@ -600,7 +601,7 @@ static bool uvRecvReleaseReq(SSvrConn* pConn, STransMsgHead* pHead) { (*pTransInst->cfp)(pTransInst->parent, &(pConn->regArg.msg), NULL); memset(&pConn->regArg, 0, sizeof(pConn->regArg)); } - uvStartSendResp(srvMsg); + uvStartSendRespInternal(srvMsg); return true; } return false; @@ -872,6 +873,7 @@ static int reallocConnRef(SSvrConn* conn) { } static void uvDestroyConn(uv_handle_t* handle) { SSvrConn* conn = handle->data; + if (conn == NULL) { return; } @@ -887,9 +889,8 @@ static void uvDestroyConn(uv_handle_t* handle) { SSvrMsg* msg = transQueueGet(&conn->srvMsgs, i); destroySmsg(msg); } - - transReqQueueClear(&conn->wreqQueue); transQueueDestroy(&conn->srvMsgs); + transReqQueueClear(&conn->wreqQueue); QUEUE_REMOVE(&conn->queue); taosMemoryFree(conn->pTcp); From 963287798dcacd32ef03b70f83f3813c006e0c67 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 17 Aug 2022 19:44:53 +0800 Subject: [PATCH 061/194] refactor(mnode): drop stream task --- include/libs/stream/tstream.h | 1 - source/dnode/mnode/impl/src/mndStb.c | 7 ++++- source/dnode/vnode/src/tq/tqSink.c | 44 +++++++++++++++------------- source/libs/stream/src/streamMeta.c | 8 +++++ source/libs/stream/src/streamQueue.c | 5 ++-- source/libs/stream/src/streamTask.c | 8 +++++ 6 files changed, 48 insertions(+), 25 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 484d0991f2..384c6a289f 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -216,7 +216,6 @@ typedef struct { void* vnode; FTbSink* tbSinkFunc; STSchema* pTSchema; - SHashObj* pHash; // groupId to tbuid } STaskSinkTb; typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 59c6d65953..6083a76981 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2044,7 +2044,12 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) break; - if (pStream->smaId == 0 && pStream->targetStbUid == suid) { + if (pStream->smaId != 0) { + sdbRelease(pSdb, pStream); + continue; + } + + if (pStream->targetStbUid == suid) { sdbRelease(pSdb, pStream); return -1; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 55630511bf..522bf46aa1 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -231,34 +231,35 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { ASSERT(pTask->tbSink.pTSchema); deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); - SSubmitReq* pReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid, - pTask->tbSink.stbFullName, &deleteReq); + SSubmitReq* submitReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid, + pTask->tbSink.stbFullName, &deleteReq); tqDebug("vgId:%d, task %d convert blocks over, put into write-queue", TD_VID(pVnode), pTask->taskId); - int32_t code; - int32_t len; - tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); - if (code < 0) { - // - ASSERT(0); - } - SEncoder encoder; - void* buf = rpcMallocCont(len + sizeof(SMsgHead)); - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncoderInit(&encoder, abuf, len); - tEncodeSBatchDeleteReq(&encoder, &deleteReq); - tEncoderClear(&encoder); - - ((SMsgHead*)buf)->vgId = pVnode->config.vgId; - if (taosArrayGetSize(deleteReq.deleteReqs) != 0) { + int32_t code; + int32_t len; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + + ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; + SRpcMsg msg = { .msgType = TDMT_VND_BATCH_DEL, - .pCont = buf, + .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead), }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(serializedDeleteReq); tqDebug("failed to put into write-queue since %s", terrstr()); } } @@ -268,11 +269,12 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { // build write msg SRpcMsg msg = { .msgType = TDMT_VND_SUBMIT, - .pCont = pReq, - .contLen = ntohl(pReq->length), + .pCont = submitReq, + .contLen = ntohl(submitReq->length), }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(submitReq); tqDebug("failed to put into write-queue since %s", terrstr()); } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index f34f68ffc6..5ff700546c 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -15,6 +15,7 @@ #include "executor.h" #include "tstream.h" +#include "ttimer.h" SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc) { SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); @@ -166,13 +167,20 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { /*return -1;*/ } + if (pTask->triggerParam != 0) { + taosTmrStop(pTask->timer); + } + while (1) { int8_t schedStatus = atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__DROPPING); if (schedStatus == TASK_SCHED_STATUS__INACTIVE) { tFreeSStreamTask(pTask); break; + } else if (schedStatus == TASK_SCHED_STATUS__DROPPING) { + break; } + taosMsleep(10); } } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 45b78a8c6e..ac10c82587 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tstream.h" +#include "streamInc.h" SStreamQueue* streamQueueOpen() { SStreamQueue* pQueue = taosMemoryCalloc(1, sizeof(SStreamQueue)); @@ -36,11 +36,12 @@ void streamQueueClose(SStreamQueue* queue) { while (1) { void* qItem = streamQueueNextItem(queue); if (qItem) { - taosFreeQitem(qItem); + streamFreeQitem(qItem); } else { break; } } taosFreeQall(queue->qall); taosCloseQueue(queue->queue); + taosMemoryFree(queue); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 0c35c1408e..d588d90543 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -156,5 +156,13 @@ void tFreeSStreamTask(SStreamTask* pTask) { if (pTask->outputQueue) streamQueueClose(pTask->outputQueue); if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg); if (pTask->exec.executor) qDestroyTask(pTask->exec.executor); + taosArrayDestroy(pTask->childEpInfo); + if (pTask->outputType == TASK_OUTPUT__TABLE) { + tDeleteSSchemaWrapper(pTask->tbSink.pSchemaWrapper); + taosMemoryFree(pTask->tbSink.pTSchema); + } + if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + taosArrayDestroy(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); + } taosMemoryFree(pTask); } From 9dc5b4cb0c532d31b65b423167eb8dd1a12a8479 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 00:08:09 +0800 Subject: [PATCH 062/194] fix(query): support read from last file. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 570 ++++++++++++++++++++----- 1 file changed, 457 insertions(+), 113 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 96567ad009..048c64032c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -37,13 +37,14 @@ typedef struct { typedef struct STableBlockScanInfo { uint64_t uid; TSKEY lastKey; - SMapData mapData; // block info (compressed) - SArray* pBlockList; // block data index list - SIterInfo iter; // mem buffer skip list iterator - SIterInfo iiter; // imem buffer skip list iterator - SArray* delSkyline; // delete info for this table + SMapData mapData; // block info (compressed) + SArray* pBlockList; // block data index list + SIterInfo iter; // mem buffer skip list iterator + SIterInfo iiter; // imem buffer skip list iterator + SArray* delSkyline; // delete info for this table int32_t fileDelIndex; - bool iterInit; // whether to initialize the in-memory skip list iterator or not + bool iterInit; // whether to initialize the in-memory skip list iterator or not + int16_t indexInBlockL;// row position in last block } STableBlockScanInfo; typedef struct SBlockOrderWrapper { @@ -76,12 +77,27 @@ typedef struct SBlockLoadSuppInfo { char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. } SBlockLoadSuppInfo; +typedef struct SVersionRange { + uint64_t minVer; + uint64_t maxVer; +} SVersionRange; + +typedef struct SLastBlockReader { + SArray* pBlockL; + int32_t currentBlockIndex; + SBlockData lastBlockData; + STimeWindow window; + SVersionRange verRange; + uint64_t uid; + int32_t rowIndex; +} SLastBlockReader; + typedef struct SFilesetIter { int32_t numOfFiles; // number of total files int32_t index; // current accessed index in the list SArray* pFileList; // data file list int32_t order; - SArray* pLastBlockList;// last block array list + SLastBlockReader* pLastBlockReader; // last file block reader } SFilesetIter; typedef struct SFileDataBlockInfo { @@ -91,14 +107,12 @@ typedef struct SFileDataBlockInfo { } SFileDataBlockInfo; typedef struct SDataBlockIter { - SBlockNumber numOfBlocks; + int32_t numOfBlocks; int32_t index; SArray* blockList; // SArray - SArray* pLastBlockList; // last block list int32_t order; SBlock block; // current SBlock data SHashObj* pTableMap; - SArray* pBlockL; // ptr to SBlockL in fileIterator } SDataBlockIter; typedef struct SFileBlockDumpInfo { @@ -108,11 +122,6 @@ typedef struct SFileBlockDumpInfo { bool allDumped; } SFileBlockDumpInfo; -typedef struct SVersionRange { - uint64_t minVer; - uint64_t maxVer; -} SVersionRange; - typedef struct SReaderStatus { bool loadFromFile; // check file stage SHashObj* pTableMap; // SHash @@ -142,7 +151,6 @@ struct STsdbReader { STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times SDataFReader* pFileReader; SVersionRange verRange; - SArray* pLastBlock; // last block info int32_t step; STsdbReader* innerReader[2]; @@ -154,10 +162,11 @@ static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i static TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader, SRowMerger* pMerger); +static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, SRowMerger* pMerger); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); -static int32_t doAppendRowFromBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, +static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); @@ -171,6 +180,8 @@ static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdb static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, int8_t* pLevel); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); +static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader); +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; @@ -191,7 +202,6 @@ static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes); - // tsdbInfo("-------------------%d\n", pCol->info.bytes); } } @@ -208,7 +218,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK } for (int32_t j = 0; j < numOfTables; ++j) { - STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; + STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid, .indexInBlockL = -1}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) { info.lastKey = pTsdbReader->window.skey; @@ -310,11 +320,15 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, int32 pIter->pFileList = aDFileSet; pIter->numOfFiles = numOfFileset; - pIter->pLastBlockList = taosArrayInit(4, sizeof(SBlockL)); - if (pIter->pLastBlockList == NULL) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), idstr); - return code; + if (pIter->pLastBlockReader == NULL) { + pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); + if (pIter->pLastBlockReader == NULL) { + int32_t code = TSDB_CODE_OUT_OF_MEMORY; + tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), idstr); + return code; + } + + pIter->pLastBlockReader->pBlockL = taosArrayInit(4, sizeof(SBlockL)); } tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, idstr); @@ -377,7 +391,7 @@ _err: static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, SHashObj* pTableMap) { pIter->order = order; pIter->index = -1; - pIter->numOfBlocks = (SBlockNumber){-1, -1}; + pIter->numOfBlocks = -1; if (pIter->blockList == NULL) { pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); } else { @@ -685,7 +699,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* } pBlockNum->numOfLastBlocks += 1; - taosArrayPush(pQualifiedLastBlock, &i); + taosArrayPush(pQualifiedLastBlock, pLastBlock); } } @@ -726,12 +740,13 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ } static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { - if (pBlockIter->index >= pBlockIter->numOfBlocks.numOfBlocks) { + if (taosArrayGetSize(pBlockIter->blockList) == 0) { + ASSERT(pBlockIter->numOfBlocks == taosArrayGetSize(pBlockIter->blockList)); return NULL; - } else { - SFileDataBlockInfo* pFBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); - return pFBlockInfo; } + + SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); + return pBlockInfo; } static SBlock* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } @@ -824,13 +839,17 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI double elapsedTime = 0; int32_t code = 0; - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - if (pFBlock != NULL) { + if (pBlockInfo != NULL) { SBlock* pBlock = getCurrentBlock(pBlockIter); code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); if (code != TSDB_CODE_SUCCESS) { + tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 + ", rows:%d, code:%s %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, + tstrerror(code), pReader->idStr); goto _error; } @@ -838,14 +857,41 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI tsdbDebug("%p load file block into buffer, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", - pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); } else { - int32_t pos = pBlockIter->index - pBlockIter->numOfBlocks.numOfBlocks; - int32_t* index = taosArrayGet(pBlockIter->pLastBlockList, pos); +#if 0 + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - SBlockL* pBlockL = taosArrayGet(pReader->status.fileIter.pLastBlockList, *index); + uint64_t uid = pBlockInfo->uid; + SArray* pBlocks = pLastBlockReader->pBlockL; + + pLastBlockReader->currentBlockIndex = -1; + + // find the correct SBlockL + for(int32_t i = 0; i < taosArrayGetSize(pBlocks); ++i) { + SBlockL* pBlock = taosArrayGet(pBlocks, i); + if (pBlock->minUid >= uid && pBlock->maxUid <= uid) { + pLastBlockReader->currentBlockIndex = i; + break; + } + } + +// SBlockL* pBlockL = taosArrayGet(pLastBlockReader->pBlockL, *index); code = tsdbReadLastBlock(pReader->pFileReader, pBlockL, pBlockData); + if (code != TSDB_CODE_SUCCESS) { + tsdbDebug("%p error occurs in loading last block into buffer, last block index:%d, total:%d brange:%" PRId64 "-%" PRId64 + ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", code:%s %s", + pReader, *index, pBlockIter->numOfBlocks.numOfLastBlocks, 0, 0, pBlockL->nRow, + pBlockL->minVer, pBlockL->maxVer, tstrerror(code), pReader->idStr); + goto _error; + } + + tsdbDebug("%p load last file block into buffer, last block index:%d, total:%d brange:%" PRId64 "-%" PRId64 + ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", + pReader, *index, pBlockIter->numOfBlocks.numOfLastBlocks, 0, 0, pBlockL->nRow, + pBlockL->minVer, pBlockL->maxVer, elapsedTime, pReader->idStr); +#endif } pReader->cost.blockLoadTime += elapsedTime; @@ -854,10 +900,6 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI return TSDB_CODE_SUCCESS; _error: -// tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 -// ", rows:%d, %s", -// pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, -// pReader->idStr); return code; } @@ -926,17 +968,12 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) { return TSDB_CODE_SUCCESS; } -static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockNumber* pBlockNum, SArray* pQLastBlock) { +static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks) { bool asc = ASCENDING_TRAVERSE(pReader->order); - pBlockIter->numOfBlocks = *pBlockNum; + pBlockIter->numOfBlocks = numOfBlocks; taosArrayClear(pBlockIter->blockList); - if (pBlockNum->numOfLastBlocks > 0) { - taosArrayDestroy(pBlockIter->pLastBlockList); - pBlockIter->pLastBlockList = pQLastBlock; - } - // access data blocks according to the offset of each block in asc/desc order. int32_t numOfTables = (int32_t)taosHashGetSize(pReader->status.pTableMap); @@ -988,21 +1025,20 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte sup.numOfTables += 1; } - ASSERT(pBlockNum->numOfBlocks == cnt); - int32_t total = pBlockNum->numOfLastBlocks + pBlockNum->numOfBlocks; + ASSERT(numOfBlocks == cnt); // since there is only one table qualified, blocks are not sorted - if (sup.numOfTables == 1 || pBlockNum->numOfBlocks == 0) { - for (int32_t i = 0; i < pBlockNum->numOfBlocks; ++i) { + if (sup.numOfTables == 1) { + for (int32_t i = 0; i < numOfBlocks; ++i) { SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i}; taosArrayPush(pBlockIter->blockList, &blockInfo); } int64_t et = taosGetTimestampUs(); tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s", - pReader, total, (et - st) / 1000.0, pReader->idStr); + pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr); - pBlockIter->index = asc ? 0 : (total - 1); + pBlockIter->index = asc ? 0 : (numOfBlocks - 1); cleanupBlockOrderSupporter(&sup); doSetCurrentBlock(pBlockIter); return TSDB_CODE_SUCCESS; @@ -1011,7 +1047,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables, pReader->idStr); - ASSERT(cnt <= pBlockNum->numOfBlocks && sup.numOfTables <= numOfTables); + ASSERT(cnt <= numOfBlocks && sup.numOfTables <= numOfTables); SMultiwayMergeTreeInfo* pTree = NULL; uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); @@ -1038,12 +1074,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } int64_t et = taosGetTimestampUs(); - tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, total, (et - st) / 1000.0, + tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr); cleanupBlockOrderSupporter(&sup); taosMemoryFree(pTree); - pBlockIter->index = asc ? 0 : (total - 1); + pBlockIter->index = asc ? 0 : (numOfBlocks - 1); doSetCurrentBlock(pBlockIter); return TSDB_CODE_SUCCESS; @@ -1053,7 +1089,7 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) { bool asc = ASCENDING_TRAVERSE(pBlockIter->order); int32_t step = asc ? 1 : -1; - if ((pBlockIter->index >= pBlockIter->numOfBlocks.numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { + if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { return false; } @@ -1100,7 +1136,7 @@ static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlock int32_t step = ASCENDING_TRAVERSE(pBlockIter->order) ? 1 : -1; int32_t index = pBlockIter->index; - while (index < pBlockIter->numOfBlocks.numOfBlocks && index >= 0) { + while (index < pBlockIter->numOfBlocks && index >= 0) { SFileDataBlockInfo* pFBlock = taosArrayGet(pBlockIter->blockList, index); if (pFBlock->uid == pFBlockInfo->uid && pFBlock->tbBlockIdx == pFBlockInfo->tbBlockIdx) { return index; @@ -1114,7 +1150,7 @@ static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlock } static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t index, int32_t step) { - if (index < 0 || index >= pBlockIter->numOfBlocks.numOfBlocks) { + if (index < 0 || index >= pBlockIter->numOfBlocks) { return -1; } @@ -1224,7 +1260,7 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBl // 4. output buffer should be large enough to hold all rows in current block // 5. delete info should not overlap with current block data static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SBlock* pBlock, - STableBlockScanInfo* pScanInfo, TSDBKEY key) { + STableBlockScanInfo* pScanInfo, TSDBKEY key, SLastBlockReader* pLastBlockReader) { int32_t neighborIndex = 0; SBlock* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order); @@ -1239,8 +1275,17 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc bool hasDup = (pBlock->nSubBlock == 1) ? pBlock->hasDup : true; bool overlapWithDel = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order); + // todo here we need to each key in the last files to identify if it is really overlapped with last block + bool overlapWithlastBlock = false; + if (hasDataInLastBlock(pLastBlockReader)) { + SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); +// int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader); + overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); + } + return (overlapWithNeighbor || hasDup || dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock) || - keyOverlapFileBlock(key, pBlock, &pReader->verRange) || (pBlock->nRow > pReader->capacity) || overlapWithDel); + keyOverlapFileBlock(key, pBlock, &pReader->verRange) || (pBlock->nRow > pReader->capacity) || + overlapWithDel || overlapWithlastBlock); } static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) { @@ -1279,7 +1324,7 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; if (nextKey != key) { // merge is not needed - doAppendRowFromBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + doAppendRowFromFileBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); pDumpInfo->rowIndex += step; return true; } @@ -1386,12 +1431,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } -static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { +static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBlockData* pBlockData = &pReader->status.fileBlockData; SArray* pDelList = pBlockScanInfo->delSkyline; TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pDelList, pReader); @@ -1532,6 +1576,14 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + // it is an multi-table data block + if (pBlockData->aUid != NULL) { + uint64_t uid = pBlockData->aUid[pDumpInfo->rowIndex]; + if (uid != pBlockScanInfo->uid) { // move to next row + return false; + } + } + // check for version and time range int64_t ver = pBlockData->aVersion[pDumpInfo->rowIndex]; if (ver > pReader->verRange.maxVer || ver < pReader->verRange.minVer) { @@ -1553,16 +1605,87 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBlockData* pBlockData = &pReader->status.fileBlockData; +static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, STimeWindow* pWin, SVersionRange* pVerRange, + int16_t startPos) { + pLastBlockReader->uid = uid; + pLastBlockReader->window = *pWin; + pLastBlockReader->verRange = *pVerRange; + pLastBlockReader->rowIndex = startPos; +} + +static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { + if (pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { + return false; + } + + pLastBlockReader->rowIndex += 1; + + SBlockData* pBlockData = &pLastBlockReader->lastBlockData; + for(int32_t i = pLastBlockReader->rowIndex; i < pBlockData->nRow; ++i) { + if (pBlockData->aUid[i] != pLastBlockReader->uid) { + continue; + } + + if (pBlockData->aTSKEY[i] < pLastBlockReader->window.skey) { + continue; + } + + if (pBlockData->aVersion[i] < pLastBlockReader->verRange.minVer) { + continue; + } + + // no data any more + if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { + return false; + } + + if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { + return false; + } + + pLastBlockReader->rowIndex = i; + return true; + } + + pLastBlockReader->rowIndex = pBlockData->nRow; + return false; +} + +static int32_t saveCurrentState(SLastBlockReader* pLastBlockReader) { + return pLastBlockReader->rowIndex; +} + +static void restoreState(SLastBlockReader* pLastBlockReader, int32_t state) { + pLastBlockReader->rowIndex = state; +} + +static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { + SBlockData* pBlockData = &pLastBlockReader->lastBlockData; + return pBlockData->aTSKEY[pLastBlockReader->rowIndex]; +} + +// todo handle desc order +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { + if (pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { + return false; + } + + return true; +} + +static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader *pLastBlockReader) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + int64_t key = INT64_MIN; + if (pBlockData->nRow > 0) { + key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + } - int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) { - return doMergeThreeLevelRows(pReader, pBlockScanInfo); + return doMergeThreeLevelRows(pReader, pBlockScanInfo, pBlockData); } else { // imem + file if (pBlockScanInfo->iiter.hasVal) { @@ -1574,18 +1697,74 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key); } - // imem & mem are all empty, only file exist - if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { - return TSDB_CODE_SUCCESS; - } else { + if (pBlockData->nRow > 0) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + // row in last file block + int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader); + if (ts < key) { // save rows in last block + SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + doMergeRowsInLastBlock(pLastBlockReader, tsLastBlock, &merge); + tRowMergerGetRow(&merge, &pTSRow); + + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; + } else if (ts == key) { + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInLastBlock(pLastBlockReader, ts, &merge); + + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; + } else { // ts > key, asc; todo handle desc + // imem & mem are all empty, only file exist + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; + } + } + } else { // only last block exists + SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + STSRow* pTSRow = NULL; SRowMerger merge = {0}; + TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInLastBlock(pLastBlockReader, tsLastBlock, &merge); tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); taosMemoryFree(pTSRow); @@ -1599,25 +1778,34 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->pResBlock; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); - STableBlockScanInfo* pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + + STableBlockScanInfo* pBlockScanInfo = NULL; + if (pBlockInfo != NULL) { + pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + } else { + pBlockScanInfo = pReader->status.pTableIter; + } + + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + + int16_t startIndex = pBlockInfo != NULL? pBlockScanInfo->indexInBlockL:-1; + initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pReader->window, &pReader->verRange, startIndex); + + bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - int32_t numOfSub = 1; - int64_t st = taosGetTimestampUs(); while (1) { // todo check the validate of row in file block { - if (!isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { + if (pBlockData->nRow > 0 && !isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { pDumpInfo->rowIndex += step; SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); - numOfSub = pBlock->nSubBlock; - if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { setBlockAllDumped(pDumpInfo, pBlock, pReader->order); break; @@ -1625,13 +1813,17 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { continue; } + + if (!hasDataInLastBlock(pLastBlockReader)) { + break; + } } - buildComposedDataBlockImpl(pReader, pBlockScanInfo); + buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); // currently loaded file data block is consumed - if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { + if (pBlockData->nRow > 0 && (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0)) { setBlockAllDumped(pDumpInfo, pBlock, pReader->order); break; } @@ -1647,9 +1839,9 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { setComposedBlockFlag(pReader, true); int64_t et = taosGetTimestampUs(); - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, subBlock:%d, brange:%" PRIu64 "-%" PRIu64 + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%d, elapsed time:%.2f ms %s", - pReader, pBlockScanInfo->uid, numOfSub, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pReader, pBlockScanInfo->uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, (et - st) / 1000.0, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -1825,7 +2017,7 @@ static TSDBKEY getCurrentKeyInBuf(SDataBlockIter* pBlockIter, STsdbReader* pRead return key; } -static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pQLastBlock) { +static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { SReaderStatus* pStatus = &pReader->status; size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); @@ -1844,21 +2036,29 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr return code; } - code = tsdbReadBlockL(pReader->pFileReader, pStatus->fileIter.pLastBlockList); + SArray* pLastBlocks = pStatus->fileIter.pLastBlockReader->pBlockL; + code = tsdbReadBlockL(pReader->pFileReader, pLastBlocks); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; } - if (taosArrayGetSize(pIndexList) > 0 || taosArrayGetSize(pStatus->fileIter.pLastBlockList) > 0) { - code = doLoadFileBlock(pReader, pIndexList, pStatus->fileIter.pLastBlockList, pBlockNum, pQLastBlock); + if (taosArrayGetSize(pIndexList) > 0 || taosArrayGetSize(pLastBlocks) > 0) { + SArray* pQLastBlock = taosArrayInit(4, sizeof(SBlockL)); + + code = doLoadFileBlock(pReader, pIndexList, pLastBlocks, pBlockNum, pQLastBlock); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); + taosArrayDestroy(pQLastBlock); return code; } if (pBlockNum->numOfBlocks + pBlockNum->numOfLastBlocks > 0) { ASSERT(taosArrayGetSize(pQLastBlock) == pBlockNum->numOfLastBlocks); + taosArrayClear(pLastBlocks); + taosArrayAddAll(pLastBlocks, pQLastBlock); + + taosArrayDestroy(pQLastBlock); break; } } @@ -1870,30 +2070,112 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr return TSDB_CODE_SUCCESS; } +static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64_t uid, STsdbReader* pReader) { + SArray* pBlocks = pLastBlockReader->pBlockL; + SBlockL* pBlock = NULL; + + pLastBlockReader->currentBlockIndex = -1; + + // find the correct SBlockL + for (int32_t i = 0; i < taosArrayGetSize(pBlocks); ++i) { + SBlockL* p = taosArrayGet(pBlocks, i); + if (p->minUid <= uid && p->maxUid >= uid) { + pLastBlockReader->currentBlockIndex = i; + pBlock = p; + break; + } + } + + if (pLastBlockReader->currentBlockIndex == -1) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = tBlockDataCreate(&pLastBlockReader->lastBlockData); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + //todo add log + return code; + } + + code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData); + if (code != TSDB_CODE_SUCCESS) { + // tsdbDebug("%p error occurs in loading last block into buffer, last block index:%d, total:%d brange:%" PRId64 "-%" PRId64 + // ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", code:%s %s", + // pReader, *index, pBlockIter->numOfBlocks.numOfLastBlocks, 0, 0, pBlock->nRow, + // pBlock->minVer, pBlock->maxVer, tstrerror(code), pReader->idStr); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; + + while(1) { + if (pStatus->pTableIter == NULL) { + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } + } else { // let's try next table + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } + } + + // find the last block that contain the specified block uid + return doLoadRelatedLastBlock(pLastBlockReader, pStatus->pTableIter->uid, pReader); + + //todo check for all empty table + } +} + static int32_t doBuildDataBlock(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - STableBlockScanInfo* pScanInfo = NULL; SBlock* pBlock = NULL; TSDBKEY key = {0}; + STableBlockScanInfo* pScanInfo = NULL; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - if (pFBlock != NULL) { - pScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); + if (pBlockInfo != NULL) { + pScanInfo = taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); pBlock = getCurrentBlock(pBlockIter); key = getCurrentKeyInBuf(pBlockIter, pReader); + + code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); + if (code != TSDB_CODE_SUCCESS) { + // todo handle error + } + + initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pReader->window, &pReader->verRange, pScanInfo->indexInBlockL); + bool hasData = nextRowInLastBlock(pLastBlockReader); + } else { + ASSERT(pBlockIter->numOfBlocks == 0); } - if (pFBlock == NULL || fileBlockShouldLoad(pReader, pFBlock, pBlock, pScanInfo, key)) { - tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClear(&pStatus->fileBlockData); + if (pBlockInfo == NULL || fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, key, pLastBlockReader)) { + if (pBlockInfo != NULL) { + tBlockDataReset(&pStatus->fileBlockData); + code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + //todo + } - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); - if (code != TSDB_CODE_SUCCESS) { - return code; + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } // build composed data block @@ -1962,22 +2244,24 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { SBlockNumber num = {0}; - SArray* pQLastBlock = taosArrayInit(4, sizeof(int32_t)); - int32_t code = moveToNextFile(pReader, &num, pQLastBlock); + int32_t code = moveToNextFile(pReader, &num); if (code != TSDB_CODE_SUCCESS) { return code; } // all data files are consumed, try data in buffer if (num.numOfBlocks + num.numOfLastBlocks == 0) { - taosArrayDestroy(pQLastBlock); pReader->status.loadFromFile = false; return code; } // initialize the block iterator for a new fileset - code = initBlockIterator(pReader, pBlockIter, &num, pQLastBlock); + if (num.numOfBlocks > 0) { + code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks); + } else { + pBlockIter->numOfBlocks = 0; + } // set the correct start position according to the query time window initBlockDumpInfo(pReader, pBlockIter); @@ -1995,6 +2279,38 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { SDataBlockIter* pBlockIter = &pReader->status.blockIter; + if (pBlockIter->numOfBlocks == 0) { + _begin: + code = doLoadLastBlockSequentially(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // all data blocks are check in last file, now let's try the next file + if (pReader->status.pTableIter == NULL) { + code = initForFirstBlockInFile(pReader, pBlockIter); + + // error happens or all the data files are completely checked + if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { + return code; + } + + // this file does not have blocks, let's start check the last block file + if (pBlockIter->numOfBlocks == 0) { + goto _begin; + } + } + + code = doBuildDataBlock(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pReader->pResBlock->info.rows > 0) { + return TSDB_CODE_SUCCESS; + } + } + while (1) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2007,17 +2323,27 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { bool hasNext = blockIteratorNext(&pReader->status.blockIter); if (hasNext) { // check for the next block in the block accessed order list initBlockDumpInfo(pReader, pBlockIter); - } else { // data blocks in current file are exhausted, let's try the next file now + } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0) { // data blocks in current file are exhausted, let's try the next file now + // todo dump all data in last block if exists. + pBlockIter->numOfBlocks = 0; + taosArrayClear(pBlockIter->blockList); + tBlockDataReset(&pReader->status.fileBlockData); + goto _begin; + } else { code = initForFirstBlockInFile(pReader, pBlockIter); // error happens or all the data files are completely checked if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { return code; } + + // this file does not have blocks, let's start check the last block file + if (pBlockIter->numOfBlocks == 0) { + goto _begin; + } } } - // current block is not loaded yet, or data in buffer may overlap with the file block. code = doBuildDataBlock(pReader); } @@ -2336,7 +2662,6 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn // 3. load the neighbor block, and set it to be the currently accessed file data block tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClear(&pStatus->fileBlockData); int32_t code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2386,6 +2711,21 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc return TSDB_CODE_SUCCESS; } +// todo support desc order +int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, SRowMerger* pMerger) { + while(nextRowInLastBlock(pLastBlockReader)) { + int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); + if (next1 == ts) { + TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, pLastBlockReader->rowIndex); + tRowMerge(pMerger, &fRow1); + } else { + break; + } + } + + return TSDB_CODE_SUCCESS; +} + void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow) { TSDBROW* pNextRow = NULL; @@ -2558,7 +2898,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex) { +int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex) { int32_t i = 0, j = 0; int32_t outputRowIndex = pResBlock->info.rows; @@ -3021,8 +3361,12 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); tBlockDataReset(&pStatus->fileBlockData); - tBlockDataClear(&pStatus->fileBlockData); - int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData); + int32_t code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pBlockScanInfo->uid, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + //todo + } + + code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData, 1); @@ -3118,12 +3462,12 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa SDataBlockIter* pBlockIter = &pStatus->blockIter; pTableBlockInfo->numOfFiles += pStatus->fileIter.numOfFiles; - if (pBlockIter->numOfBlocks.numOfBlocks > 0) { - pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks.numOfBlocks; + if (pBlockIter->numOfBlocks > 0) { + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; } pTableBlockInfo->numOfTables = numOfTables; - bool hasNext = (pBlockIter->numOfBlocks.numOfBlocks > 0); + bool hasNext = (pBlockIter->numOfBlocks > 0); while (true) { if (hasNext) { @@ -3154,8 +3498,8 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa break; } - pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks.numOfBlocks; - hasNext = (pBlockIter->numOfBlocks.numOfBlocks > 0); + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; + hasNext = (pBlockIter->numOfBlocks > 0); } // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, From e7e6a4c7e4669f45ae2eca690c7bda57d115a398 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 18 Aug 2022 09:22:00 +0800 Subject: [PATCH 063/194] fix invalid packet --- source/libs/transport/src/transSvr.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index f84b87a43d..f50711c59c 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -284,6 +284,9 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } } return; + } else { + destroyConn(conn, true); + return; } } if (nread == 0) { From 5f54f90416dbf31021a75c30cc7f11e0c2fc2c93 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 18 Aug 2022 09:30:24 +0800 Subject: [PATCH 064/194] fix: empty struct --- include/common/tmsg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index cedbdb6087..8948d9f670 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2664,7 +2664,7 @@ typedef struct { } SVgEpSet; typedef struct { - // padding + int64_t padding; } SRSmaExecMsg; typedef struct { From 4bc57e96ba4b2c4b8d0861543e84a9a0d86af8e6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 18 Aug 2022 09:40:58 +0800 Subject: [PATCH 065/194] fix invalid packet --- source/libs/transport/src/thttp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 806a786b00..76a7611c2e 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -16,10 +16,10 @@ #define _DEFAULT_SOURCE // clang-format off #include +#include "zlib.h" #include "thttp.h" #include "taoserror.h" #include "tlog.h" -#include "zlib.h" typedef struct SHttpClient { uv_connect_t conn; @@ -196,8 +196,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 uv_loop_t* loop = uv_default_loop(); uv_tcp_init(loop, &cli->tcp); // set up timeout to avoid stuck; - //int32_t fd = taosCreateSocketWithTimeout(5); - //uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); + int32_t fd = taosCreateSocketWithTimeout(5); + uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); int32_t ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&dest, clientConnCb); if (ret != 0) { From c5d3440b8419abc8b34964a0777acd5e06d9bb56 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 18 Aug 2022 09:58:24 +0800 Subject: [PATCH 066/194] other: code optimization --- include/common/tmsg.h | 2 +- source/dnode/vnode/src/inc/sma.h | 12 +++++++----- source/dnode/vnode/src/vnd/vnodeCommit.c | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index cedbdb6087..f870bd161f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2664,7 +2664,7 @@ typedef struct { } SVgEpSet; typedef struct { - // padding + int32_t padding; } SRSmaExecMsg; typedef struct { diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 26adc8d5e5..c43772062e 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -118,17 +118,19 @@ struct SSmaStat { #define RSMA_FS_LOCK(r) (&(r)->lock) struct SRSmaInfoItem { - int8_t level; - int8_t triggerStat; - int32_t maxDelay; - tmr_h tmrId; + int8_t level; + int8_t triggerStat; + uint16_t interval; // second + int32_t maxDelay; + tmr_h tmrId; }; struct SRSmaInfo { STSchema *pTSchema; int64_t suid; int64_t refId; // refId of SRSmaStat - int8_t delFlag; + uint64_t delFlag : 1; + uint64_t lastReceived : 63; // second T_REF_DECLARE() SRSmaInfoItem items[TSDB_RETENTION_L2]; void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index fc09eaac44..89542baa25 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -220,8 +220,6 @@ int vnodeCommit(SVnode *pVnode) { vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64, TD_VID(pVnode), pVnode->state.commitID, pVnode->state.applied); - pVnode->state.commitTerm = pVnode->state.applyTerm; - // preCommit // smaSyncPreCommit(pVnode->pSma); smaAsyncPreCommit(pVnode->pSma); @@ -229,6 +227,8 @@ int vnodeCommit(SVnode *pVnode) { vnodeBufPoolUnRef(pVnode->inUse); pVnode->inUse = NULL; + pVnode->state.commitTerm = pVnode->state.applyTerm; + // save info info.config = pVnode->config; info.state.committed = pVnode->state.applied; From 64747cb029bbecd31d414c7eab8fe992ec8701b2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 18 Aug 2022 10:02:27 +0800 Subject: [PATCH 067/194] other: code optimization --- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 89542baa25..64f223b974 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -228,7 +228,7 @@ int vnodeCommit(SVnode *pVnode) { pVnode->inUse = NULL; pVnode->state.commitTerm = pVnode->state.applyTerm; - + // save info info.config = pVnode->config; info.state.committed = pVnode->state.applied; From 7e73f3617303e37f7d2d5123c90eb51ac1f821be Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 18 Aug 2022 10:08:22 +0800 Subject: [PATCH 068/194] other: code format revert --- source/libs/executor/inc/executil.h | 2 +- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index d5b979c762..52c73f85f5 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -140,4 +140,4 @@ int32_t resultrowComparAsc(const void* p1, const void* p2); int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); -#endif // TDENGINE_QUERYUTIL_H \ No newline at end of file +#endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 11d371d49b..311d82c8a2 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -1045,4 +1045,4 @@ void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); } #endif -#endif // TDENGINE_EXECUTORIMPL_H \ No newline at end of file +#endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 615d742d40..34247d3b47 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -986,4 +986,4 @@ void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimit pLimitInfo->slimit = slimit; pLimitInfo->remainOffset = limit.offset; pLimitInfo->remainGroupOffset = slimit.offset; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4380a662fe..d15dc99122 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4777,4 +4777,4 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF pCtx[i].pBuf = pSup->pResultBuf; } return code; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c26f468238..8c4fbe7971 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3134,4 +3134,4 @@ _error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); return NULL; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b81cb7724f..6418f5305c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -5487,4 +5487,4 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; -} \ No newline at end of file +} From 418810fca5695fdf2403036ca5d12353ec050401 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 10:09:31 +0800 Subject: [PATCH 069/194] optimize bitmap --- include/common/tdataformat.h | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 3679b3773b..93a693cea9 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -38,22 +38,22 @@ typedef struct STagVal STagVal; typedef struct STag STag; // bitmap -#define N1(n) ((1 << (n)) - 1) -#define BIT1_SIZE(n) (((n)-1) / 8 + 1) -#define BIT2_SIZE(n) (((n)-1) / 4 + 1) -#define SET_BIT1(p, i, v) \ - do { \ - (p)[(i) / 8] &= N1((i) % 8); \ - (p)[(i) / 8] |= (((uint8_t)(v)) << (((i) % 8))); \ - } while (0) +const static uint8_t BIT1_MAP[8][2] = { + {0b00000000, 0b00000001}, {0b00000001, 0b00000011}, {0b00000011, 0b00000111}, {0b00000111, 0b00001111}, + {0b00001111, 0b00011111}, {0b00011111, 0b00111111}, {0b00111111, 0b01111111}, {0b01111111, 0b11111111}, +}; -#define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) \ - do { \ - p[(i) / 4] &= N1((i) % 4 * 2); \ - (p)[(i) / 4] |= (((uint8_t)(v)) << (((i) % 4) * 2)); \ - } while (0) -#define GET_BIT2(p, i) (((p)[(i) / 4] >> (((i) % 4) * 2)) & ((uint8_t)3)) +const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, + {0b00000011, 0b00000111, 0b00001011, 2}, + {0b00001111, 0b00011111, 0b00101111, 4}, + {0b00111111, 0b01111111, 0b10111111, 6}}; + +#define BIT1_SIZE(n) (((n)-1) / 8 + 1) +#define BIT2_SIZE(n) (((n)-1) / 4 + 1) +#define SET_BIT1(p, i, v) ((p)[(i) / 8] &= BIT1_MAP[(i) % 8][v]) +#define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) +#define SET_BIT2(p, i, v) ((p)[(i) / 4] &= BIT2_MAP[(i) % 4][v]) +#define GET_BIT2(p, i) (((p)[(i) / 4] >> BIT2_MAP[(i) % 4][3]) & ((uint8_t)3)) // STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); @@ -171,7 +171,7 @@ struct SColVal { #pragma pack(push, 1) struct STagVal { -// char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta + // char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta union { int16_t cid; char *pKey; From 3276b62aba782dfea13a43b941051854894986b8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 10:29:49 +0800 Subject: [PATCH 070/194] more --- include/common/tdataformat.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 93a693cea9..bc29f3b59c 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -48,12 +48,12 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, {0b00001111, 0b00011111, 0b00101111, 4}, {0b00111111, 0b01111111, 0b10111111, 6}}; -#define BIT1_SIZE(n) (((n)-1) / 8 + 1) -#define BIT2_SIZE(n) (((n)-1) / 4 + 1) -#define SET_BIT1(p, i, v) ((p)[(i) / 8] &= BIT1_MAP[(i) % 8][v]) -#define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) ((p)[(i) / 4] &= BIT2_MAP[(i) % 4][v]) -#define GET_BIT2(p, i) (((p)[(i) / 4] >> BIT2_MAP[(i) % 4][3]) & ((uint8_t)3)) +#define BIT1_SIZE(n) (((n)-1) >> 3 + 1) +#define BIT2_SIZE(n) (((n)-1) >> 2 + 1) +#define SET_BIT1(p, i, v) ((p)[(i) >> 3] &= BIT1_MAP[(i)&7][v]) +#define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) +#define SET_BIT2(p, i, v) ((p)[(i) >> 2] &= BIT2_MAP[(i)&3][v]) +#define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) // STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); From 4dc5df8453da96264097d857e88211212aa68197 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 18 Aug 2022 10:34:55 +0800 Subject: [PATCH 071/194] fix invalid packet --- source/libs/transport/src/transSvr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index be6eafba97..10063df329 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -277,7 +277,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { tTrace("%s conn %p alread read complete packet", transLabel(pTransInst), conn); if (uvHandleReq(conn) == false) { destroyConn(conn, true); - break; + return; } } return; From 1ecf660c575939968e86954770a12d8e4d11aa66 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 10:45:52 +0800 Subject: [PATCH 072/194] other:merge 3.0 --- source/client/CMakeLists.txt | 4 +-- source/client/test/clientTests.cpp | 48 ++++++++++++++---------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index f52edbe71f..95e86e7629 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -46,6 +46,6 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) -if(${BUILD_TEST}) +#if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) \ No newline at end of file +#endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index ec270889e2..4ea5443678 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -123,7 +123,7 @@ void createNewTable(TAOS* pConn, int32_t index) { } taos_free_result(pRes); - for(int32_t i = 0; i < 100000; i += 20) { + for(int32_t i = 0; i < 3280; i += 20) { char sql[1024] = {0}; sprintf(sql, "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" @@ -679,30 +679,28 @@ TEST(testCase, projection_query_tables) { TAOS_RES* pRes = taos_query(pConn, "use abc1"); taos_free_result(pRes); - pRes = taos_query(pConn, "explain verbose true select _wstart,count(*),a from st1 partition by a interval(1s)"); - printResult(pRes); -// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tu using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// for(int32_t i = 0; i < 1; ++i) { -// printf("create table :%d\n", i); -// createNewTable(pConn, i); -// } + pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + for(int32_t i = 0; i < 2; ++i) { + printf("create table :%d\n", i); + createNewTable(pConn, i); + } // // pRes = taos_query(pConn, "select * from tu"); // if (taos_errno(pRes) != 0) { From e70cfbbd002eb0fae6e68011c901e08802b26675 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 10:58:39 +0800 Subject: [PATCH 073/194] fix(query): add jmp macro define. --- source/libs/executor/src/executorimpl.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d15dc99122..6134e9eafa 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -76,6 +76,12 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define realloc u_realloc #endif +#define T_LONG_JMP(_obj, _c) \ + do { \ + assert((_c) != -1); \ + longjmp((_obj), (_c)); \ + } while (0); + #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) #define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) From b4c2dc137ce36e310a14d240f25798e718a6aa24 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 11:15:31 +0800 Subject: [PATCH 074/194] fix coredump --- include/common/tdataformat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index bc29f3b59c..e94edfcb1e 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -48,8 +48,8 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, {0b00001111, 0b00011111, 0b00101111, 4}, {0b00111111, 0b01111111, 0b10111111, 6}}; -#define BIT1_SIZE(n) (((n)-1) >> 3 + 1) -#define BIT2_SIZE(n) (((n)-1) >> 2 + 1) +#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) +#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) #define SET_BIT1(p, i, v) ((p)[(i) >> 3] &= BIT1_MAP[(i)&7][v]) #define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) #define SET_BIT2(p, i, v) ((p)[(i) >> 2] &= BIT2_MAP[(i)&3][v]) From e702730790b5f6381cbfe8b0fd5e3fc67c8168eb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 11:36:11 +0800 Subject: [PATCH 075/194] fix: coredump --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 9fcfd9c8bd..fd39d559ea 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1198,7 +1198,6 @@ _exit: int32_t tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom) { int32_t code = 0; - ASSERT(0); ASSERT(pBlockDataFrom->suid || pBlockDataFrom->uid); pBlockData->suid = pBlockDataFrom->suid; @@ -1437,7 +1436,6 @@ int32_t tBlockDataCopy(SBlockData *pSrc, SBlockData *pDest) { ASSERT(pDest->suid == pSrc->suid); ASSERT(pDest->uid == pSrc->uid); - ASSERT(pSrc->nRow == pDest->nRow); ASSERT(taosArrayGetSize(pSrc->aIdx) == taosArrayGetSize(pDest->aIdx)); pDest->nRow = pSrc->nRow; From 1a449a51a1215c428e818d54e506487e194de966 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 18 Aug 2022 12:05:56 +0800 Subject: [PATCH 076/194] fix invalid packet --- source/libs/transport/src/thttp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 76a7611c2e..b902fbd13a 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -178,7 +178,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 } terrno = 0; - char header[1024] = {0}; + char header[2048] = {0}; int32_t headLen = taosBuildHttpHeader(server, contLen, header, sizeof(header), flag); uv_buf_t* wb = taosMemoryCalloc(2, sizeof(uv_buf_t)); From fb2bb0481ec9bb66f2e7b3f9213ff44b883aa709 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 18 Aug 2022 13:22:59 +0800 Subject: [PATCH 077/194] fix(query): fix interp pResBlock->rows > capacity cause assert failure --- source/libs/executor/src/timewindowoperator.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 3e931a489d..757ab09d1a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2154,9 +2154,7 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; - if (rows >= pResBlock->info.capacity) { - return; - } + blockDataEnsureCapacity(pResBlock, rows + 1); // todo set the correct primary timestamp column // output the result From f3ec1593bcd85eadff34d141e4825e58314c3b86 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 18 Aug 2022 13:55:56 +0800 Subject: [PATCH 078/194] fix invalid packet --- source/libs/transport/src/transSvr.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 10063df329..753ea00a45 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -265,10 +265,10 @@ static bool uvHandleReq(SSvrConn* pConn) { } void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { - // opt - SSvrConn* conn = cli->data; + SSvrConn* conn = cli->data; + STrans* pTransInst = conn->pTransInst; + SConnBuffer* pBuf = &conn->readBuf; - STrans* pTransInst = conn->pTransInst; if (nread > 0) { pBuf->len += nread; tTrace("%s conn %p total read:%d, current read:%d", transLabel(pTransInst), conn, pBuf->len, (int)nread); From e1ae5bc7f8884898e96c0a661614facf6cd5f350 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 18 Aug 2022 13:59:23 +0800 Subject: [PATCH 079/194] fix invalid packet --- source/common/src/tglobal.c | 6 +++--- source/libs/transport/src/thttp.c | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c763bbed9c..adc5af1a17 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -75,7 +75,7 @@ int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; // telem -bool tsEnableTelem = false; +bool tsEnableTelem = true; int32_t tsTelemInterval = 86400; char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com"; uint16_t tsTelemPort = 80; @@ -166,7 +166,7 @@ int32_t tsTtlPushInterval = 86400; int32_t tsGrantHBInterval = 60; #ifndef _STORAGE -int32_t taosSetTfsCfg(SConfig *pCfg) { +int32_t taosSetTfsCfg(SConfig *pCfg) { SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); memset(tsDataDir, 0, PATH_MAX); @@ -180,7 +180,7 @@ int32_t taosSetTfsCfg(SConfig *pCfg) { uError("failed to create dataDir:%s", tsDataDir); return -1; } - return 0; + return 0; } #else int32_t taosSetTfsCfg(SConfig *pCfg); diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index b902fbd13a..2b5c56965f 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -152,15 +152,15 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { } static int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { - uint32_t ipv4 = taosGetIpv4FromFqdn(server); - if (ipv4 == 0xffffffff) { + uint32_t ip = taosGetIpv4FromFqdn(server); + if (ip == 0xffffffff) { terrno = TAOS_SYSTEM_ERROR(errno); uError("http-report failed to get http server:%s ip since %s", server, terrstr()); return -1; } - char ipv4Buf[128] = {0}; - tinet_ntoa(ipv4Buf, ipv4); - uv_ip4_addr(ipv4Buf, port, dest); + char buf[128] = {0}; + tinet_ntoa(buf, ip); + uv_ip4_addr(buf, port, dest); return 0; } int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { From 15110480bc3839889c851b45353f678b2b0ad3bf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 18 Aug 2022 14:22:07 +0800 Subject: [PATCH 080/194] fix invalid packet --- source/libs/transport/src/transSvr.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 753ea00a45..c980b70abd 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -204,7 +204,6 @@ static bool uvHandleReq(SSvrConn* pConn) { // uv_read_stop((uv_stream_t*)pConn->pTcp); // transRefSrvHandle(pConn); // uv_queue_work(((SWorkThrd*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask); - if (uvRecvReleaseReq(pConn, pHead)) { return true; } From a2757eb4f381ff4dc7b561fa1ca90caa572aa90a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 18 Aug 2022 14:26:11 +0800 Subject: [PATCH 081/194] fix(mnode): memory leak --- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/inc/mndStb.h | 1 + source/dnode/mnode/impl/src/mndDef.c | 19 +++++++++++++++++++ source/dnode/mnode/impl/src/mndStb.c | 16 ++++++++++------ source/dnode/mnode/impl/src/mndStream.c | 13 ++++++++++++- 5 files changed, 43 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 455da6a40e..8cff7fe48e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -636,6 +636,7 @@ typedef struct { int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj); +void tFreeStreamObj(SStreamObj* pObj); typedef struct { char streamName[TSDB_STREAM_FNAME_LEN]; diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 44a7fdadde..010199a89f 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -34,6 +34,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate); SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName); int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb); int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb); +void mndFreeStb(SStbObj *pStb); void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst); void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 08ce161409..e6f1a40993 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -116,6 +116,25 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { return 0; } +void tFreeStreamObj(SStreamObj *pStream) { + taosMemoryFree(pStream->sql); + taosMemoryFree(pStream->ast); + taosMemoryFree(pStream->physicalPlan); + if (pStream->outputSchema.nCols) taosMemoryFree(pStream->outputSchema.pSchema); + + int32_t sz = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < sz; i++) { + SArray *pLevel = taosArrayGetP(pStream->tasks, i); + int32_t taskSz = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < taskSz; j++) { + SStreamTask *pTask = taosArrayGetP(pLevel, j); + tFreeSStreamTask(pTask); + } + taosArrayDestroy(pLevel); + } + taosArrayDestroy(pStream->tasks); +} + SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); if (pVgEpNew == NULL) return NULL; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 6083a76981..e0f2b83160 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -266,6 +266,15 @@ _OVER: return pRow; } +void mndFreeStb(SStbObj *pStb) { + taosArrayDestroy(pStb->pFuncs); + taosMemoryFreeClear(pStb->pColumns); + taosMemoryFreeClear(pStb->pTags); + taosMemoryFreeClear(pStb->comment); + taosMemoryFreeClear(pStb->pAst1); + taosMemoryFreeClear(pStb->pAst2); +} + static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) { mTrace("stb:%s, perform insert action, row:%p", pStb->name, pStb); return 0; @@ -273,12 +282,7 @@ static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) { static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { mTrace("stb:%s, perform delete action, row:%p", pStb->name, pStb); - taosArrayDestroy(pStb->pFuncs); - taosMemoryFreeClear(pStb->pColumns); - taosMemoryFreeClear(pStb->pTags); - taosMemoryFreeClear(pStb->comment); - taosMemoryFreeClear(pStb->pAst1); - taosMemoryFreeClear(pStb->pAst2); + mndFreeStb(pStb); return 0; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 8c453e0c88..6dc8e2072b 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -167,6 +167,9 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream) { static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) { mTrace("stream:%s, perform delete action", pStream->name); + taosWLockLatch(&pStream->lock); + tFreeStreamObj(pStream); + taosWUnLockLatch(&pStream->lock); return 0; } @@ -493,10 +496,17 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre stbObj.uid = pStream->targetStbUid; - if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; + if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) { + mndFreeStb(&stbObj); + goto _OVER; + } + + tFreeSMCreateStbReq(&createReq); + mndFreeStb(&stbObj); return 0; _OVER: + tFreeSMCreateStbReq(&createReq); mndReleaseStb(pMnode, pStb); mndReleaseDb(pMnode, pDb); return -1; @@ -715,6 +725,7 @@ _OVER: mndReleaseDb(pMnode, pDb); tFreeSCMCreateStreamReq(&createStreamReq); + tFreeStreamObj(&streamObj); return code; } From 45dc2ec1f20f9e1df21b6f044bfb4cc116b6bdb0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 18 Aug 2022 14:32:11 +0800 Subject: [PATCH 082/194] fix(tsdb/cache): add DCLP to tsdbCache --- source/dnode/vnode/src/inc/tsdb.h | 3 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 53 ++++++++++++++++--------- source/dnode/vnode/src/tsdb/tsdbOpen.c | 2 +- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f1e980c026..a30f308ecd 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -260,7 +260,7 @@ void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap); // tsdbCache int32_t tsdbOpenCache(STsdb *pTsdb); -void tsdbCloseCache(SLRUCache *pCache); +void tsdbCloseCache(STsdb *pTsdb); int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb); int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup); int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h); @@ -298,6 +298,7 @@ struct STsdb { SMemTable *imem; STsdbFS fs; SLRUCache *lruCache; + TdThreadMutex lruMutex; }; struct TSDBKEY { diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index f03b02af27..1d2c5c3b32 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -33,16 +33,21 @@ int32_t tsdbOpenCache(STsdb *pTsdb) { taosLRUCacheSetStrictCapacity(pCache, true); + taosThreadMutexInit(&pTsdb->lruMutex, NULL); + _err: pTsdb->lruCache = pCache; return code; } -void tsdbCloseCache(SLRUCache *pCache) { +void tsdbCloseCache(STsdb *pTsdb) { + SLRUCache *pCache = pTsdb->lruCache; if (pCache) { taosLRUCacheEraseUnrefEntries(pCache); taosLRUCacheCleanup(pCache); + + taosThreadMutexDestroy(&pTsdb->lruMutex); } } @@ -1100,26 +1105,38 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH // getTableCacheKeyS(uid, "lr", key, &keyLen); getTableCacheKey(uid, 0, key, &keyLen); LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - } else { - STSRow *pRow = NULL; - bool dup = false; // which is always false for now - code = mergeLastRow(uid, pTsdb, &dup, &pRow); - // if table's empty or error, return code of -1 - if (code < 0 || pRow == NULL) { - if (!dup && pRow) { - taosMemoryFree(pRow); + if (!h) { + taosThreadMutexLock(&pTsdb->lruMutex); + + h = taosLRUCacheLookup(pCache, key, keyLen); + if (!h) { + STSRow *pRow = NULL; + bool dup = false; // which is always false for now + code = mergeLastRow(uid, pTsdb, &dup, &pRow); + // if table's empty or error, return code of -1 + if (code < 0 || pRow == NULL) { + if (!dup && pRow) { + taosMemoryFree(pRow); + } + + *handle = NULL; + return 0; } - *handle = NULL; - return 0; - } + _taos_lru_deleter_t deleter = deleteTableCacheLastrow; + LRUStatus status = + taosLRUCacheInsert(pCache, key, keyLen, pRow, TD_ROW_LEN(pRow), deleter, NULL, TAOS_LRU_PRIORITY_LOW); + if (status != TAOS_LRU_STATUS_OK) { + code = -1; + } - _taos_lru_deleter_t deleter = deleteTableCacheLastrow; - LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, pRow, TD_ROW_LEN(pRow), deleter, NULL, TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; + taosThreadMutexUnlock(&pTsdb->lruMutex); + } else { + taosThreadMutexUnlock(&pTsdb->lruMutex); + + *handle = h; + + return code; } h = taosLRUCacheLookup(pCache, key, keyLen); diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index be2828d187..ec760e3c57 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -86,7 +86,7 @@ int tsdbClose(STsdb **pTsdb) { if (*pTsdb) { taosThreadRwlockDestroy(&(*pTsdb)->rwLock); tsdbFSClose(*pTsdb); - tsdbCloseCache((*pTsdb)->lruCache); + tsdbCloseCache(*pTsdb); taosMemoryFreeClear(*pTsdb); } return 0; From a4e4f4cc4a7c515a9f760d568d379ce3e605eac4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 14:37:43 +0800 Subject: [PATCH 083/194] make it compile --- source/client/CMakeLists.txt | 4 ++-- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 16 +++++----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 95e86e7629..f52edbe71f 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -46,6 +46,6 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) -#if(${BUILD_TEST}) +if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -#endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a490393b4c..f03f7e2a9b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -41,9 +41,9 @@ struct STsdbSnapReader { SArray* aDelData; // SArray }; -#if 0 static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; +#if 0 STsdb* pTsdb = pReader->pTsdb; while (true) { @@ -166,6 +166,7 @@ _exit: _err: tsdbError("vgId:%d, vnode snapshot tsdb read data for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); +#endif return code; } @@ -180,7 +181,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { } // open - code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb, NULL); + code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb); if (code) goto _err; // read index @@ -253,11 +254,9 @@ _err: tstrerror(code)); return code; } -#endif int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) { - int32_t code = 0; -#if 0 + int32_t code = 0; STsdbSnapReader* pReader = NULL; // alloc @@ -320,13 +319,11 @@ _err: tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); *ppReader = NULL; -#endif return code; } int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { - int32_t code = 0; -#if 0 + int32_t code = 0; STsdbSnapReader* pReader = *ppReader; if (pReader->pDataFReader) { @@ -349,13 +346,11 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { taosMemoryFree(pReader); *ppReader = NULL; -#endif return code; } int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; -#if 0 *ppData = NULL; @@ -394,7 +389,6 @@ _exit: _err: tsdbError("vgId:%d, vnode snapshot tsdb read for %s failed since %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path, tstrerror(code)); -#endif return code; } From 8a32c0c1898bc6a90d6bbe73592eb44f15f0c70f Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 18 Aug 2022 14:55:57 +0800 Subject: [PATCH 084/194] refactor(sync): adjust strategy for dynamic quorum --- include/libs/sync/sync.h | 1 + source/libs/sync/inc/syncInt.h | 2 +- source/libs/sync/src/syncCommit.c | 40 ++++++++++++++++++++++++------- source/libs/sync/src/syncMain.c | 22 ++++++++++++++--- 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 952066df46..790cbf906d 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -34,6 +34,7 @@ extern bool gRaftDetailLog; #define SYNC_MAX_PROGRESS_WAIT_MS 4000 #define SYNC_MAX_START_TIME_RANGE_MS (1000 * 20) #define SYNC_MAX_RECV_TIME_RANGE_MS 1000 +#define SYNC_ADD_QUORUM_COUNT 3 #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index de43c81654..0afc373f2d 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -237,7 +237,7 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode); bool syncNodeHasSnapshot(SSyncNode* pSyncNode); void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode); -SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode); +SyncIndex syncNodeGetLastIndex(const SSyncNode* pSyncNode); SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode); int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 3829ea0730..1e68fe346c 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -146,23 +146,47 @@ int32_t syncNodeDynamicQuorum(const SSyncNode* pSyncNode) { int64_t timeNow = taosGetTimestampMs(); for (int i = 0; i < pSyncNode->peersNum; ++i) { - int64_t peerStartTime = syncIndexMgrGetStartTime(pSyncNode->pNextIndex, &(pSyncNode->peersId)[i]); - int64_t peerRecvTime = syncIndexMgrGetRecvTime(pSyncNode->pNextIndex, &(pSyncNode->peersId)[i]); + int64_t peerStartTime = syncIndexMgrGetStartTime(pSyncNode->pNextIndex, &(pSyncNode->peersId)[i]); + int64_t peerRecvTime = syncIndexMgrGetRecvTime(pSyncNode->pNextIndex, &(pSyncNode->peersId)[i]); + SyncIndex peerMatchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId)[i]); - int64_t recvTimeDiff = syncNodeAbs64(peerRecvTime, timeNow); - int64_t startTimeDiff = syncNodeAbs64(peerStartTime, pSyncNode->startTime); + int64_t recvTimeDiff = TABS(peerRecvTime - timeNow); + int64_t startTimeDiff = TABS(peerStartTime - pSyncNode->startTime); + int64_t logDiff = TABS(peerMatchIndex - syncNodeGetLastIndex(pSyncNode)); + + /* + int64_t recvTimeDiff = syncNodeAbs64(peerRecvTime, timeNow); + int64_t startTimeDiff = syncNodeAbs64(peerStartTime, pSyncNode->startTime); + int64_t logDiff = syncNodeAbs64(peerMatchIndex, syncNodeGetLastIndex(pSyncNode)); + */ int32_t addQuorum = 0; if (recvTimeDiff < SYNC_MAX_RECV_TIME_RANGE_MS) { - addQuorum = 1; + if (startTimeDiff < SYNC_MAX_START_TIME_RANGE_MS) { + addQuorum = 1; + } else { + if (logDiff < SYNC_ADD_QUORUM_COUNT) { + addQuorum = 1; + } else { + addQuorum = 0; + } + } } else { addQuorum = 0; } - if (startTimeDiff > SYNC_MAX_START_TIME_RANGE_MS) { - addQuorum = 0; - } + /* + if (recvTimeDiff < SYNC_MAX_RECV_TIME_RANGE_MS) { + addQuorum = 1; + } else { + addQuorum = 0; + } + + if (startTimeDiff > SYNC_MAX_START_TIME_RANGE_MS) { + addQuorum = 0; + } + */ quorum += addQuorum; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index a00b59d292..3fe600ecbb 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2284,7 +2284,7 @@ bool syncNodeHasSnapshot(SSyncNode* pSyncNode) { // return max(logLastIndex, snapshotLastIndex) // if no snapshot and log, return -1 -SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) { +SyncIndex syncNodeGetLastIndex(const SSyncNode* pSyncNode) { SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); @@ -2773,11 +2773,27 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p return 0; } - if (ths->vgId > 1) { - syncNodeEventLog(ths, "I am vnode, can not do leader transfer"); + if (pEntry->term < ths->pRaftStore->currentTerm) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "little term:%lu, can not do leader transfer", pEntry->term); + syncNodeEventLog(ths, logBuf); return 0; } + if (pEntry->index < syncNodeGetLastIndex(ths)) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "little index:%ld, can not do leader transfer", pEntry->index); + syncNodeEventLog(ths, logBuf); + return 0; + } + + /* + if (ths->vgId > 1) { + syncNodeEventLog(ths, "I am vnode, can not do leader transfer"); + return 0; + } + */ + do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), "do leader transfer, index:%ld", pEntry->index); From 117bc90dad8fb295513e82891a6c7d1728993975 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 15:29:42 +0800 Subject: [PATCH 085/194] more snapshot --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 103 ++++++++++----------- 2 files changed, 50 insertions(+), 54 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0c7a08a2b5..4d3a074963 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -368,6 +368,7 @@ struct SSma { void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); enum { + SNAP_DATA_CFG = 0, SNAP_DATA_META = 1, SNAP_DATA_TSDB = 2, SNAP_DATA_DEL = 3, diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index f03f7e2a9b..6ba546ecf1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -354,6 +354,7 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { *ppData = NULL; +#if 0 // read data file if (!pReader->dataDone) { code = tsdbSnapReadData(pReader, ppData); @@ -367,6 +368,7 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { } } } +#endif // read del file if (!pReader->delDone) { @@ -956,6 +958,7 @@ _err: tstrerror(code)); return code; } +#endif static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; @@ -966,7 +969,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 // reader if (pDelFile) { - code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL); + code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb); if (code) goto _err; code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR); @@ -983,51 +986,15 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); while (true) { - SDelIdx* pDelIdx = NULL; - int64_t n = sizeof(SSnapDataHdr) + sizeof(TABLEID); - SDelData delData; - SDelIdx delIdx; - int8_t toBreak = 0; + if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break; + if (tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) >= 0) break; - if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR)) { - pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); - } + SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); - if (pDelIdx) { - int32_t c = tTABLEIDCmprFn(&id, pDelIdx); - if (c < 0) { - goto _new_del; - } else { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); - if (code) goto _err; + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); + if (code) goto _err; - pWriter->iDelIdx++; - if (c == 0) { - toBreak = 1; - delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid}; - goto _merge_del; - } else { - delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid}; - goto _write_del; - } - } - } - - _new_del: - toBreak = 1; - delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid}; - taosArrayClear(pWriter->aDelData); - - _merge_del: - while (n < nData) { - n += tGetDelData(pData + n, &delData); - if (taosArrayPush(pWriter->aDelData, &delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - - _write_del: + SDelIdx delIdx = *pDelIdx; code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); if (code) goto _err; @@ -1036,7 +1003,40 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 goto _err; } - if (toBreak) break; + pWriter->iDelIdx++; + } + + if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR) && + tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) == 0) { + SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); + + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); + if (code) goto _err; + + pWriter->iDelIdx++; + } else { + taosArrayClear(pWriter->aDelData); + } + + int64_t n = sizeof(SSnapDataHdr) + sizeof(TABLEID); + while (n < nData) { + SDelData delData; + + n += tGetDelData(pData + n, &delData); + + if (taosArrayPush(pWriter->aDelData, &delData) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + + SDelIdx delIdx = {.suid = id.suid, .uid = id.uid}; + code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); + if (code) goto _err; + + if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } _exit: @@ -1047,9 +1047,7 @@ _err: tstrerror(code)); return code; } -#endif -#if 0 static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; @@ -1062,7 +1060,7 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); if (code) goto _err; - SDelIdx delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid}; + SDelIdx delIdx = *pDelIdx; code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); if (code) goto _err; @@ -1095,11 +1093,9 @@ _err: tstrerror(code)); return code; } -#endif int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { - int32_t code = 0; -#if 0 + int32_t code = 0; STsdbSnapWriter* pWriter = NULL; // alloc @@ -1168,7 +1164,6 @@ _err: tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); *ppWriter = NULL; -#endif return code; } @@ -1210,10 +1205,10 @@ _err: } int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; -#if 0 + int32_t code = 0; SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; +#if 0 // ts data if (pHdr->type == SNAP_DATA_TSDB) { code = tsdbSnapWriteData(pWriter, pData, nData); @@ -1226,6 +1221,7 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) if (code) goto _err; } } +#endif // del data if (pHdr->type == SNAP_DATA_DEL) { @@ -1241,6 +1237,5 @@ _exit: _err: tsdbError("vgId:%d, tsdb snapshot write for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path, tstrerror(code)); -#endif return code; } From d9e978814ad66b87478a88e53a506afe57a02a88 Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:33:12 +0800 Subject: [PATCH 086/194] ci: rust dir The 2.x and 3.x have the same file name but different contents. --- Jenkinsfile2 | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 754617f99f..12e806c87a 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -43,6 +43,7 @@ def pre_test(){ cd ${WKC} git reset --hard git clean -fxd + rm -rf examples/rust/ git remote prune origin git fetch ''' From 03bb9e7f016f3004a34f5ce56e97de5729b7f480 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 Aug 2022 15:44:28 +0800 Subject: [PATCH 087/194] fix: get meta null if table does not exists --- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/meta/metaQuery.c | 9 +++++++++ source/libs/executor/src/executil.c | 9 +++++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index c224f6ce7f..e39863721e 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -95,6 +95,7 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHash int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); +bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); typedef struct SMetaFltParam { tb_uid_t suid; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 4e85cde7ca..7cf365d372 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -127,6 +127,15 @@ _err: // return 0; // } +bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { + // query uid.idx + if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { + return false; + } + + return true; +} + int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { SMeta *pMeta = pReader->pMeta; int64_t version; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1dabea0d6b..baeb972e05 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -533,7 +533,9 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, vnodeGetCtbIdList(pVnode, pScanNode->suid, res); } } else { // Create one table group. - taosArrayPush(res, &tableUid); + if(metaIsTableExist(metaHandle, tableUid)){ + taosArrayPush(res, &tableUid); + } } if (pTagCond) { @@ -599,7 +601,10 @@ size_t getTableTagsBufLen(const SNodeList* pGroups) { int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) { SMetaReader mr = {0}; metaReaderInit(&mr, pMeta, 0); - metaGetTableEntryByUid(&mr, uid); + if(metaGetTableEntryByUid(&mr, uid) != 0){ // table not exist + metaReaderClear(&mr); + return TSDB_CODE_PAR_TABLE_NOT_EXIST; + } SNodeList* groupNew = nodesCloneList(pGroupNode); From 082d2093b8e5dc11e6e6abecf0d980888fde2be0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 16:00:24 +0800 Subject: [PATCH 088/194] open more --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 6ba546ecf1..81c5190a32 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -354,7 +354,6 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { *ppData = NULL; -#if 0 // read data file if (!pReader->dataDone) { code = tsdbSnapReadData(pReader, ppData); @@ -368,7 +367,6 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { } } } -#endif // read del file if (!pReader->delDone) { From 9ef2d736897bb0c0feb81980cb259a31a43ef957 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 18 Aug 2022 16:06:49 +0800 Subject: [PATCH 089/194] fix(query): fix elapsed order by ts desc result inconsist after flush to disk TD-18453 --- source/libs/function/src/builtinsimpl.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5aeaecd598..bf4a07f8e2 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3970,16 +3970,16 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) { TSKEY* ptsList = (int64_t*)colDataGetData(pCol, 0); if (pCtx->order == TSDB_ORDER_DESC) { if (pCtx->start.key == INT64_MIN) { - pInfo->max = - (pInfo->max < ptsList[start + pInput->numOfRows - 1]) ? ptsList[start + pInput->numOfRows - 1] : pInfo->max; + pInfo->max = (pInfo->max < ptsList[start]) ? ptsList[start] : pInfo->max; } else { pInfo->max = pCtx->start.key + 1; } - if (pCtx->end.key != INT64_MIN) { - pInfo->min = pCtx->end.key; + if (pCtx->end.key == INT64_MIN) { + pInfo->min = (pInfo->min > ptsList[start + pInput->numOfRows - 1]) ? + ptsList[start + pInput->numOfRows - 1] : pInfo->min; } else { - pInfo->min = ptsList[start]; + pInfo->min = pCtx->end.key; } } else { if (pCtx->start.key == INT64_MIN) { @@ -3988,10 +3988,11 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) { pInfo->min = pCtx->start.key; } - if (pCtx->end.key != INT64_MIN) { - pInfo->max = pCtx->end.key + 1; + if (pCtx->end.key == INT64_MIN) { + pInfo->max = (pInfo->max < ptsList[start + pInput->numOfRows - 1]) ? + ptsList[start + pInput->numOfRows - 1] : pInfo->max; } else { - pInfo->max = ptsList[start + pInput->numOfRows - 1]; + pInfo->max = pCtx->end.key + 1; } } } From 13a93d68ebdddc3d27ce39fa1962586f044b70aa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 18 Aug 2022 16:09:46 +0800 Subject: [PATCH 090/194] fix invalid packet --- source/libs/transport/src/thttp.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 62277a7569..dc0f9128b0 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -221,7 +221,6 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 uError("failed to receive response from %s:%u since %s", server, port, terrstr()); goto SEND_OVER; } - code = 0; SEND_OVER: From 7c2d6571f043680ad32bea7d477a741c27e7f716 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 18 Aug 2022 16:30:54 +0800 Subject: [PATCH 091/194] refactor(sync): adjust SYNC_MAX_RECV_TIME_RANGE_MS --- include/libs/sync/sync.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 790cbf906d..e6a4dd1d49 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -33,7 +33,7 @@ extern bool gRaftDetailLog; #define SYNC_MAX_READ_RANGE 2 #define SYNC_MAX_PROGRESS_WAIT_MS 4000 #define SYNC_MAX_START_TIME_RANGE_MS (1000 * 20) -#define SYNC_MAX_RECV_TIME_RANGE_MS 1000 +#define SYNC_MAX_RECV_TIME_RANGE_MS 1200 #define SYNC_ADD_QUORUM_COUNT 3 #define SYNC_MAX_BATCH_SIZE 1 From df2224988f486fa5672fa4a44146a05bc41be3cc Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 18 Aug 2022 17:02:52 +0800 Subject: [PATCH 092/194] fix(query): restrict interp input type to numerical type only TD-18487 --- source/libs/function/src/builtins.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index c19b459cdb..ed82e4cb50 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1503,11 +1503,17 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } + uint8_t nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, 0)); + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(paraType) || QUERY_NODE_VALUE == nodeType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + if (3 <= numOfParams) { int64_t timeVal[2] = {0}; for (int32_t i = 1; i < 3; ++i) { - uint8_t nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, i)); - uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type; + nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, i)); + paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type; if (!IS_VAR_DATA_TYPE(paraType) || QUERY_NODE_VALUE != nodeType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1525,8 +1531,8 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } if (4 == numOfParams) { - uint8_t nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, 3)); - uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type; + nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, 3)); + paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type; if (!IS_INTEGER_TYPE(paraType) || QUERY_NODE_VALUE != nodeType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } From 4de48e66de8a84812e6d769436c6f03e19d574a2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 17:04:24 +0800 Subject: [PATCH 093/194] adjust more --- source/dnode/vnode/src/inc/tsdb.h | 1 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 1 + source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 37 +++++++++++++------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 + 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 6263703c01..f41592fe46 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -502,6 +502,7 @@ struct SDelIdx { struct SDiskDataHdr { uint32_t delimiter; + uint32_t fmtVer; int64_t suid; int64_t uid; int32_t szUid; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 324db5b111..55ef576916 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1426,6 +1426,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock // ================= DATA ==================== SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, + .fmtVer = 0, .suid = pBlockData->suid, .uid = pBlockData->uid, .nRow = pBlockData->nRow, diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 81c5190a32..29a64cf0b5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -27,8 +27,10 @@ struct STsdbSnapReader { int32_t fid; SDataFReader* pDataFReader; SArray* aBlockIdx; // SArray + SArray* aBlockL; // SArray int32_t iBlockIdx; SBlockIdx* pBlockIdx; + int32_t iBlockL; SMapData mBlock; // SMapData int32_t iBlock; SBlockData oBlockData; @@ -43,32 +45,37 @@ struct STsdbSnapReader { static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; -#if 0 STsdb* pTsdb = pReader->pTsdb; while (true) { if (pReader->pDataFReader == NULL) { - SDFileSet* pSet = - taosArraySearch(pReader->fs.aDFileSet, &(SDFileSet){.fid = pReader->fid}, tDFileSetCmprFn, TD_GT); - + // next + SDFileSet dFileSet = {.fid = pReader->fid}; + SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT); if (pSet == NULL) goto _exit; - pReader->fid = pSet->fid; + + // load code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); if (code) goto _err; - // SBlockIdx code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); if (code) goto _err; + code = tsdbReadBlockL(pReader->pDataFReader, pReader->aBlockL); + if (code) goto _err; + + // init pReader->iBlockIdx = 0; pReader->pBlockIdx = NULL; + pReader->iBlockL = 0; tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path, pReader->fid); } while (true) { +#if 0 if (pReader->pBlockIdx == NULL) { if (pReader->iBlockIdx >= taosArrayGetSize(pReader->aBlockIdx)) { tsdbDataFReaderClose(&pReader->pDataFReader); @@ -149,14 +156,15 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { tPutBlockData((uint8_t*)(&pId[1]), &pReader->nBlockData); - tsdbInfo("vgId:%d, vnode snapshot read data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64 - " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d size:%d", - TD_VID(pTsdb->pVnode), pTsdb->path, pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid, - pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow, - size); + // tsdbInfo("vgId:%d, vnode snapshot read data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64 + // " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d size:%d", + // TD_VID(pTsdb->pVnode), pTsdb->path, pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid, + // pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow, + // size); goto _exit; } +#endif } } @@ -166,7 +174,6 @@ _exit: _err: tsdbError("vgId:%d, vnode snapshot tsdb read data for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); -#endif return code; } @@ -294,6 +301,11 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + pReader->aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pReader->aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } pReader->mBlock = tMapDataInit(); code = tBlockDataCreate(&pReader->oBlockData); if (code) goto _err; @@ -329,6 +341,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { if (pReader->pDataFReader) { tsdbDataFReaderClose(&pReader->pDataFReader); } + taosArrayDestroy(pReader->aBlockL); taosArrayDestroy(pReader->aBlockIdx); tMapDataClear(&pReader->mBlock); tBlockDataDestroy(&pReader->oBlockData, 1); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index fd39d559ea..f324124a9c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1549,6 +1549,7 @@ int32_t tPutDiskDataHdr(uint8_t *p, void *ph) { SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; n += tPutU32(p ? p + n : p, pHdr->delimiter); + n += tPutU32v(p ? p + n : p, pHdr->fmtVer); n += tPutI64(p ? p + n : p, pHdr->suid); n += tPutI64(p ? p + n : p, pHdr->uid); n += tPutI32v(p ? p + n : p, pHdr->szUid); @@ -1566,6 +1567,7 @@ int32_t tGetDiskDataHdr(uint8_t *p, void *ph) { SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; n += tGetU32(p + n, &pHdr->delimiter); + n += tGetU32v(p + n, &pHdr->fmtVer); n += tGetI64(p + n, &pHdr->suid); n += tGetI64(p + n, &pHdr->uid); n += tGetI32v(p + n, &pHdr->szUid); From 8eb86676d091badf9946de6cbc285bd53446d0e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 17:09:44 +0800 Subject: [PATCH 094/194] fix(query): support query last file. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 173 ++++++++++++++++++++++--- 1 file changed, 156 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 048c64032c..4af8d6e28f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1279,7 +1279,6 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc bool overlapWithlastBlock = false; if (hasDataInLastBlock(pLastBlockReader)) { SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); -// int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader); overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); } @@ -1358,8 +1357,71 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* return pReader->pMemSchema; } +// todo handle desc +static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, + SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { + SRowMerger merge = {0}; + STSRow* pTSRow = NULL; + SBlockData* pBlockData = &pReader->status.fileBlockData; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + + SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + + int64_t minKey = INT64_MAX; + if (minKey > tsLast) { + minKey = tsLast; + } + + if (minKey > k.ts) { + minKey = k.ts; + } + + if (minKey > key && pBlockData->nRow > 0) { + minKey = key; + } + + // file block ---> last block -----> imem -----> mem + bool init = false; + if (minKey == key) { + init = true; + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + } + + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } + + if (minKey == k.ts) { + if (!init) { + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, pRow, pSchema); + } + + doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; +} + static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, - SIterInfo* pIter, int64_t key) { + SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; SBlockData* pBlockData = &pReader->status.fileBlockData; @@ -1431,6 +1493,85 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } +static int32_t doMergeThreeLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { + SRowMerger merge = {0}; + STSRow* pTSRow = NULL; + + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + SArray* pDelList = pBlockScanInfo->delSkyline; + + TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pDelList, pReader); + TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pDelList, pReader); + ASSERT(pRow != NULL && piRow != NULL); + + SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + + int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + + int64_t minKey = INT64_MAX; + if (minKey > k.ts) { + minKey = k.ts; + } + + if (minKey > ik.ts) { + minKey = ik.ts; + } + + if (minKey > key) { + minKey = key; + } + + if (minKey > tsLast) { + minKey = tsLast; + } + + // file block ---> last block -----> imem -----> mem + bool init = false; + if (minKey == key) { + init = true; + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + } + + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } + + if (minKey == ik.ts) { + if (!init) { + init = true; + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, piRow, pSchema); + } + doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + if (minKey == k.ts) { + if (!init) { + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, pRow, pSchema); + } + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; +} + static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; @@ -1605,8 +1746,8 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, STimeWindow* pWin, SVersionRange* pVerRange, - int16_t startPos) { +static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, STimeWindow* pWin, + SVersionRange* pVerRange, int16_t startPos) { pLastBlockReader->uid = uid; pLastBlockReader->window = *pWin; pLastBlockReader->verRange = *pVerRange; @@ -1636,10 +1777,12 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { // no data any more if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { + pLastBlockReader->rowIndex = pBlockData->nRow; return false; } if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { + pLastBlockReader->rowIndex = pBlockData->nRow; return false; } @@ -1676,25 +1819,21 @@ static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader *pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t key = INT64_MIN; - if (pBlockData->nRow > 0) { - key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; - } - + int64_t key = (pBlockData->nRow > 0)? pBlockData->aTSKEY[pDumpInfo->rowIndex]:INT64_MIN; TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) { - return doMergeThreeLevelRows(pReader, pBlockScanInfo, pBlockData); + return doMergeThreeLevelRowsRv(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); } else { - // imem + file + // imem + file + last block if (pBlockScanInfo->iiter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key); + return doMergeBufAndFileRows_Rv(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader); } // mem + file if (pBlockScanInfo->iter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key); + return doMergeBufAndFileRows_Rv(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader); } if (pBlockData->nRow > 0) { @@ -1704,7 +1843,6 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader); if (ts < key) { // save rows in last block SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); STSRow* pTSRow = NULL; SRowMerger merge = {0}; @@ -1712,7 +1850,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); - doMergeRowsInLastBlock(pLastBlockReader, tsLastBlock, &merge); + doMergeRowsInLastBlock(pLastBlockReader, ts, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); @@ -1753,6 +1891,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } } } else { // only last block exists + // only last block exits SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); @@ -1820,10 +1959,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); - SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); // currently loaded file data block is consumed - if (pBlockData->nRow > 0 && (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0)) { + if (pBlockData->nRow > 0 && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { + SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); setBlockAllDumped(pDumpInfo, pBlock, pReader->order); break; } From 48bf21d321b923e3f1331b2f984189f1a1785c39 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 18 Aug 2022 17:32:08 +0800 Subject: [PATCH 095/194] build: move check into shell file --- cmake/cmake.install | 32 +----------------------- contrib/CMakeLists.txt | 18 -------------- packaging/tools/make_install.bat | 42 +++++++++++++++++++++++++++++++- packaging/tools/make_install.sh | 4 ++- 4 files changed, 45 insertions(+), 51 deletions(-) diff --git a/cmake/cmake.install b/cmake/cmake.install index 4e3d0b166a..6dc6864975 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -1,38 +1,8 @@ -IF (EXISTS /var/lib/taos/dnode/dnodeCfg.json) - INSTALL(CODE "MESSAGE(\"The default data directory /var/lib/taos contains old data of tdengine 2.x, please clear it before installing!\")") -ELSEIF (EXISTS C:/TDengine/data/dnode/dnodeCfg.json) - INSTALL(CODE "MESSAGE(\"The default data directory C:/TDengine/data contains old data of tdengine 2.x, please clear it before installing!\")") -ELSEIF (TD_LINUX) +IF (TD_LINUX) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") INSTALL(CODE "execute_process(COMMAND bash ${TD_MAKE_INSTALL_SH} ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Linux ${TD_VER_NUMBER})") ELSEIF (TD_WINDOWS) - SET(CMAKE_INSTALL_PREFIX C:/TDengine) - - # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go DESTINATION connector) - # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector) - # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector) - # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector) - # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .) - INSTALL(CODE "IF (NOT EXISTS ${CMAKE_INSTALL_PREFIX}/cfg/taos.cfg) - execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg ${CMAKE_INSTALL_PREFIX}/cfg/taos.cfg) - ENDIF ()") - INSTALL(FILES ${TD_SOURCE_DIR}/include/client/taos.h DESTINATION include) - INSTALL(FILES ${TD_SOURCE_DIR}/include/util/taoserror.h DESTINATION include) - INSTALL(FILES ${TD_SOURCE_DIR}/include/libs/function/taosudf.h DESTINATION include) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos_static.lib DESTINATION driver) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) - INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) - INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosd.exe DESTINATION .) - INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/udfd.exe DESTINATION .) - IF (BUILD_TOOLS) - INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosBenchmark.exe DESTINATION .) - ENDIF () - - IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.38-dist.jar DESTINATION connector/jdbc) - ENDIF () SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.bat") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER})") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index a1eec81ee0..494d1577fc 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -135,24 +135,6 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download") - -# clear submodule -execute_process(COMMAND git submodule deinit -f tools/taos-tools - WORKING_DIRECTORY "${TD_SOURCE_DIR}") -execute_process(COMMAND git rm --cached tools/taos-tools - WORKING_DIRECTORY "${TD_SOURCE_DIR}") -execute_process(COMMAND git submodule deinit -f tools/taosadapter - WORKING_DIRECTORY "${TD_SOURCE_DIR}") -execute_process(COMMAND git rm --cached tools/taosadapter - WORKING_DIRECTORY "${TD_SOURCE_DIR}") -execute_process(COMMAND git submodule deinit -f tools/taosws-rs - WORKING_DIRECTORY "${TD_SOURCE_DIR}") -execute_process(COMMAND git rm --cached tools/taosws-rs - WORKING_DIRECTORY "${TD_SOURCE_DIR}") -execute_process(COMMAND git submodule deinit -f examples/rust - WORKING_DIRECTORY "${TD_SOURCE_DIR}") -execute_process(COMMAND git rm --cached examples/rust - WORKING_DIRECTORY "${TD_SOURCE_DIR}") # ================================================================================================ # Build diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index 0f9e836ae2..3c27c1beca 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -1,7 +1,47 @@ @echo off goto %1 :needAdmin + +if exist C:\\TDengine\\data\\dnode\\dnodeCfg.json ( + echo The default data directory C:/TDengine/data contains old data of tdengine 2.x, please clear it before installing! +) +set source_dir=%2 +set source_dir=%source_dir:/=\\% +set binary_dir=%3 +set binary_dir=%binary_dir:/=\\% +set osType=%4 +set verNumber=%5 +set tagert_dir=C:\\TDengine + +if not exist %tagert_dir% ( + mkdir %tagert_dir% +) +if not exist %tagert_dir%\\cfg ( + mkdir %tagert_dir%\\cfg +) +if not exist %tagert_dir%\\include ( + mkdir %tagert_dir%\\include +) +if not exist %tagert_dir%\\driver ( + mkdir %tagert_dir%\\driver +) +if not exist C:\\TDengine\\cfg\\taos.cfg ( + copy %source_dir%\\packaging\\cfg\\taos.cfg %tagert_dir%\\cfg\\taos.cfg > nul +) +copy %source_dir%\\include\\client\\taos.h %tagert_dir%\\include > nul +copy %source_dir%\\include\\util\\taoserror.h %tagert_dir%\\include > nul +copy %source_dir%\\include\\libs\\function\\taosudf.h %tagert_dir%\\include > nul +copy %binary_dir%\\build\\lib\\taos.lib %tagert_dir%\\driver > nul +copy %binary_dir%\\build\\lib\\taos_static.lib %tagert_dir%\\driver > nul +copy %binary_dir%\\build\\lib\\taos.dll %tagert_dir%\\driver > nul +copy %binary_dir%\\build\\bin\\taos.exe %tagert_dir% > nul +copy %binary_dir%\\build\\bin\\taosd.exe %tagert_dir% > nul +copy %binary_dir%\\build\\bin\\udfd.exe %tagert_dir% > nul +if exist %binary_dir%\\build\\bin\\taosBenchmark.exe ( + copy %binary_dir%\\build\\bin\\taosBenchmark.exe %tagert_dir% > nul +) + mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)&& echo To start/stop TDengine with administrator privileges: sc start/stop taosd &goto :eof :hasAdmin -cp -f C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 +copy /y C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 > nul sc query "taosd" >nul || sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index d8d4c5bf2a..6a95ace99e 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -664,7 +664,9 @@ function install_TDengine() { ## ==============================Main program starts from here============================ echo source directory: $1 echo binary directory: $2 -if [ "$osType" != "Darwin" ]; then +if [ -x ${data_dir}/dnode/dnodeCfg.json ]; then + echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of tdengine 2.x, please clear it before installing!\033[0m" +elif [ "$osType" != "Darwin" ]; then if [ -x ${bin_dir}/${clientName} ]; then update_TDengine else From 452395bc46b1d15e5f24f7eb31a3351387365f1b Mon Sep 17 00:00:00 2001 From: xleili Date: Thu, 18 Aug 2022 17:33:28 +0800 Subject: [PATCH 096/194] fix(docs):uncomment nodejs tmq example's commit --- docs/examples/node/nativeexample/subscribe_demo.js | 3 +-- docs/examples/node/package.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/examples/node/nativeexample/subscribe_demo.js b/docs/examples/node/nativeexample/subscribe_demo.js index 5b65e1c907..53cbe55d26 100644 --- a/docs/examples/node/nativeexample/subscribe_demo.js +++ b/docs/examples/node/nativeexample/subscribe_demo.js @@ -28,8 +28,7 @@ function runConsumer() { console.log(msg.topicPartition); console.log(msg.block); console.log(msg.fields) - // fixme(@xiaolei): commented temp, should be fixed. - //consumer.commit(msg); + consumer.commit(msg); console.log(`=======consumer ${i} done`) } diff --git a/docs/examples/node/package.json b/docs/examples/node/package.json index 36d3f016b5..d00d71d99f 100644 --- a/docs/examples/node/package.json +++ b/docs/examples/node/package.json @@ -4,7 +4,7 @@ "main": "index.js", "license": "MIT", "dependencies": { - "@tdengine/client": "^3.0.0", + "@tdengine/client": "^3.0.1", "@tdengine/rest": "^3.0.0" } } From 3e6ccb83570052d781d85e491dcf9695e3be5478 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 Aug 2022 18:00:03 +0800 Subject: [PATCH 097/194] fix: filter producing zero rows will not be treated as sys table scan operator completion --- source/libs/executor/src/scanoperator.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 367fae66a6..7da290240f 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2213,9 +2213,18 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { colDataAppend(pColInfoData, numOfRows, n, false); if (++numOfRows >= pOperator->resultInfo.capacity) { - break; + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); + doFilterResult(pInfo); + + blockDataCleanup(p); + if (pInfo->pRes->info.rows > 0) + break; } } + blockDataDestroy(p); // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found if (ret != 0) { @@ -2224,14 +2233,6 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); } - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); - doFilterResult(pInfo); - - blockDataDestroy(p); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } From 0bfadd871931dc18f90deb6b843e8ad35feb9a14 Mon Sep 17 00:00:00 2001 From: BoDing Date: Thu, 18 Aug 2022 18:22:17 +0800 Subject: [PATCH 098/194] fix broken link to taosx --- docs/zh/17-operation/03-tolerance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/17-operation/03-tolerance.md b/docs/zh/17-operation/03-tolerance.md index 2cfd4b6484..6ee6aa6373 100644 --- a/docs/zh/17-operation/03-tolerance.md +++ b/docs/zh/17-operation/03-tolerance.md @@ -27,4 +27,4 @@ TDengine 集群的节点数必须大于等于副本数，否则创建表时将 当 TDengine 集群中的节点部署在不同的物理机上，并设置多个副本数时，就实现了系统的高可靠性，无需再使用其他软件或工具。TDengine 企业版还可以将副本部署在不同机房，从而实现异地容灾。 -另外一种灾备方式是通过 `taosX` 将一个 TDengine 集群的数据同步复制到物理上位于不同数据中心的另一个 TDengine 集群。其详细使用方法请参考 [taosX 参考手册](../../reference/taosX) +另外一种灾备方式是通过 `taosX` 将一个 TDengine 集群的数据同步复制到物理上位于不同数据中心的另一个 TDengine 集群。其详细使用方法请参考 [taosX 参考手册](../../reference/taosx) From 98fd9aca109e9d791e87dd05f9a72d6b0c22bbf6 Mon Sep 17 00:00:00 2001 From: BoDing Date: Thu, 18 Aug 2022 18:25:01 +0800 Subject: [PATCH 099/194] remove link to taosx --- docs/zh/17-operation/03-tolerance.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/zh/17-operation/03-tolerance.md b/docs/zh/17-operation/03-tolerance.md index 6ee6aa6373..1ce485b042 100644 --- a/docs/zh/17-operation/03-tolerance.md +++ b/docs/zh/17-operation/03-tolerance.md @@ -26,5 +26,3 @@ TDengine 集群中的时序数据的副本数是与数据库关联的，一个 TDengine 集群的节点数必须大于等于副本数，否则创建表时将报错。 当 TDengine 集群中的节点部署在不同的物理机上，并设置多个副本数时，就实现了系统的高可靠性，无需再使用其他软件或工具。TDengine 企业版还可以将副本部署在不同机房，从而实现异地容灾。 - -另外一种灾备方式是通过 `taosX` 将一个 TDengine 集群的数据同步复制到物理上位于不同数据中心的另一个 TDengine 集群。其详细使用方法请参考 [taosX 参考手册](../../reference/taosx) From 871b4895f92dd74af5559dbfc838e2db09cd3963 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 18:26:10 +0800 Subject: [PATCH 100/194] fix bit map bug --- include/common/tdataformat.h | 4 ++-- source/dnode/vnode/src/tsdb/tsdbUtil.c | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index e94edfcb1e..1cc226f9d7 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -50,9 +50,9 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, #define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) #define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) ((p)[(i) >> 3] &= BIT1_MAP[(i)&7][v]) +#define SET_BIT1(p, i, v) ((p)[(i) >> 3] |= BIT1_MAP[(i)&7][v]) #define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) ((p)[(i) >> 2] &= BIT2_MAP[(i)&3][v]) +#define SET_BIT2(p, i, v) ((p)[(i) >> 2] |= BIT2_MAP[(i)&3][v]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) // STSchema diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index f324124a9c..ad759f1ef8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -928,6 +928,9 @@ int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { size = BIT2_SIZE(pColData->nVal + 1); code = tRealloc(&pColData->pBitMap, size); if (code) goto _exit; + if ((pColData->nVal & 3) == 0) { + pColData->pBitMap[pColData->nVal >> 2] = 0; + } // put value if (pColVal->isNone) { From 1a8fb961823be1ed14f05852155700fe2f002040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 18 Aug 2022 18:30:03 +0800 Subject: [PATCH 101/194] fix: reset row number when filter produces zero row --- source/libs/executor/src/scanoperator.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index f2c4a8e3b6..e578e34b82 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2220,8 +2220,12 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { doFilterResult(pInfo); blockDataCleanup(p); - if (pInfo->pRes->info.rows > 0) + if (pInfo->pRes->info.rows > 0) { break; + } else { + numOfRows = 0; + continue; + } } } blockDataDestroy(p); From 2b092f6551988aebd4aaf6938363593b999979ed Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 18 Aug 2022 18:39:21 +0800 Subject: [PATCH 102/194] fix(query): memory leak --- source/dnode/mnode/impl/src/mndSma.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/libs/executor/src/executorimpl.c | 36 +++++++++---------- source/libs/executor/src/scanoperator.c | 2 ++ source/libs/executor/src/timewindowoperator.c | 6 +++- source/libs/stream/src/streamTask.c | 3 +- 6 files changed, 29 insertions(+), 22 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 006d9e749c..6db9b59c69 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -489,7 +489,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); ASSERT(smaObj.uid != 0); char resultTbName[TSDB_TABLE_FNAME_LEN + 16] = {0}; - snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "%s_td_tsma_rst_tb",pCreate->name); + snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "%s_td_tsma_rst_tb", pCreate->name); memcpy(smaObj.dstTbName, resultTbName, TSDB_TABLE_FNAME_LEN); smaObj.dstTbUid = mndGenerateUid(smaObj.dstTbName, TSDB_TABLE_FNAME_LEN); smaObj.stbUid = pStb->uid; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index b8803b20fc..e6a331f20e 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -341,7 +341,7 @@ FAIL: return -1; } -void tqReaderSetColIdList(STqReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } +void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; } int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { if (pReader->tbIdHash) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d15dc99122..9ff5b5d759 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3217,8 +3217,8 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa } static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); -static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo *pOperator, SFillOperatorInfo* pInfo, SResultInfo* pResultInfo, - SExecTaskInfo* pTaskInfo) { +static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, + SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; SSDataBlock* pResBlock = pInfo->pFinalRes; @@ -3242,8 +3242,8 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo *pOperator, SFillOp pInfo->existNewGroupBlock = NULL; } -static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, SResultInfo* pResultInfo, - SExecTaskInfo* pTaskInfo) { +static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, + SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows; taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows); @@ -3259,8 +3259,8 @@ static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOpera static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) { SFillOperatorInfo* pInfo = pOperator->info; - SExprSupp* pSup = &pOperator->exprSupp; - SSDataBlock* pResBlock = pInfo->pFinalRes; + SExprSupp* pSup = &pOperator->exprSupp; + SSDataBlock* pResBlock = pInfo->pFinalRes; setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, scanFlag, false); projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, NULL); @@ -3270,13 +3270,13 @@ static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlo SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, pInfo->primarySrcSlotId); colDataAssign(pDst, pSrc, pInfo->pRes->info.rows, &pResBlock->info); - for(int32_t i = 0; i < pInfo->numOfNotFillExpr; ++i) { + for (int32_t i = 0; i < pInfo->numOfNotFillExpr; ++i) { SFillColInfo* pCol = &pInfo->pFillInfo->pFillCol[i + pInfo->numOfExpr]; ASSERT(pCol->notFillCol); SExprInfo* pExpr = pCol->pExpr; - int32_t srcSlotId = pExpr->base.pParam[0].pCol->slotId; - int32_t dstSlotId = pExpr->base.resSchema.slotId; + int32_t srcSlotId = pExpr->base.pParam[0].pCol->slotId; + int32_t dstSlotId = pExpr->base.resSchema.slotId; SColumnInfoData* pDst1 = taosArrayGet(pInfo->pRes->pDataBlock, dstSlotId); SColumnInfoData* pSrc1 = taosArrayGet(pBlock->pDataBlock, srcSlotId); @@ -3664,7 +3664,7 @@ void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { taosRemoveRef(exchangeObjRefPool, pExInfo->self); } -void freeSourceDataInfo(void *p) { +void freeSourceDataInfo(void* p) { SSourceDataInfo* pInfo = (SSourceDataInfo*)p; taosMemoryFreeClear(pInfo->pRsp); } @@ -3694,8 +3694,8 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, win.skey); w = getFirstQualifiedTimeWindow(win.skey, &w, pInterval, TSDB_ORDER_ASC); - pInfo->pFillInfo = - taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); + pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, + pInfo->primaryTsCol, order, id); pInfo->win = win; pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); @@ -3721,10 +3721,10 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr); pInfo->pNotFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pInfo->numOfNotFillExpr); - SInterval* pInterval = + SInterval* pInterval = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType - ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval - : &((SIntervalAggOperatorInfo*)downstream->info)->interval; + ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval + : &((SIntervalAggOperatorInfo*)downstream->info)->interval; int32_t order = (pPhyFillNode->inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; int32_t type = convertFillType(pPhyFillNode->mode); @@ -3741,9 +3741,9 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); - int32_t code = - initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr, (SNodeListNode*)pPhyFillNode->pValues, - pPhyFillNode->timeRange, pResultInfo->capacity, pTaskInfo->id.str, pInterval, type, order); + int32_t code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr, + (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity, + pTaskInfo->id.str, pInterval, type, order); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index aca0078592..d116533fe5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1649,6 +1649,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } taosArrayDestroy(tableIdList); memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->cond, sizeof(SQueryTableDataCond)); + } else { + taosArrayDestroy(pColIds); } // create the pseduo columns info diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 757ab09d1a..02be01cdb8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1706,14 +1706,19 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { blockDataDestroy(pInfo->pPullDataRes); taosArrayDestroy(pInfo->pRecycledPages); blockDataDestroy(pInfo->pUpdateRes); + taosArrayDestroy(pInfo->pDelWins); + blockDataDestroy(pInfo->pDelRes); if (pInfo->pChildren) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i); destroyStreamFinalIntervalOperatorInfo(pChildOp->info, numOfOutput); + taosMemoryFree(pChildOp->pDownstream); + cleanupExprSupp(&pChildOp->exprSupp); taosMemoryFreeClear(pChildOp); } + taosArrayDestroy(pInfo->pChildren); } nodesDestroyNode((SNode*)pInfo->pPhyNode); colDataDestroy(&pInfo->twAggSup.timeWindowData); @@ -2644,7 +2649,6 @@ void destroyTimeSliceOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(param); } - SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index d588d90543..4009a47c65 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -152,11 +152,12 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { } void tFreeSStreamTask(SStreamTask* pTask) { + qDebug("free stream task %d", pTask->taskId); if (pTask->inputQueue) streamQueueClose(pTask->inputQueue); if (pTask->outputQueue) streamQueueClose(pTask->outputQueue); if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg); if (pTask->exec.executor) qDestroyTask(pTask->exec.executor); - taosArrayDestroy(pTask->childEpInfo); + taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree); if (pTask->outputType == TASK_OUTPUT__TABLE) { tDeleteSSchemaWrapper(pTask->tbSink.pSchemaWrapper); taosMemoryFree(pTask->tbSink.pTSchema); From e75c4000454e70f1e68c5d1fc73f3762d673150c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 18:48:50 +0800 Subject: [PATCH 103/194] fix(query): fix bug in query last block. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 155 +++++++++++++++---------- 1 file changed, 93 insertions(+), 62 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bc20240b4c..f4671a0f34 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -89,7 +89,7 @@ typedef struct SLastBlockReader { STimeWindow window; SVersionRange verRange; uint64_t uid; - int32_t rowIndex; + int16_t* rowIndex; // row index ptr, usually from the STableBlockScanInfo->indexInBlockL } SLastBlockReader; typedef struct SFilesetIter { @@ -181,7 +181,8 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* re int8_t* pLevel); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader); -static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); +static int32_t doBuildDataBlock(STsdbReader* pReader); static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; @@ -449,7 +450,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level); pReader->suid = pCond->suid; pReader->order = pCond->order; - pReader->capacity = capacity; + pReader->capacity = 1; pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; @@ -624,24 +625,29 @@ _end: return code; } -static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* pLastBlockIndex, - SBlockNumber * pBlockNum, SArray* pQualifiedLastBlock) { - int32_t numOfQTable= 0; - size_t numOfTables = taosArrayGetSize(pIndexList); - - int64_t st = taosGetTimestampUs(); - size_t size = 0; - +static void cleanupTableScanInfo(SHashObj* pTableMap) { STableBlockScanInfo* px = NULL; while (1) { - px = taosHashIterate(pReader->status.pTableMap, px); + px = taosHashIterate(pTableMap, px); if (px == NULL) { break; } + // reset the index in last block when handing a new file + px->indexInBlockL = -1; tMapDataClear(&px->mapData); taosArrayClear(px->pBlockList); } +} + +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* pLastBlockIndex, + SBlockNumber * pBlockNum, SArray* pQualifiedLastBlock) { + int32_t numOfQTable = 0; + size_t sizeInDisk = 0; + size_t numOfTables = taosArrayGetSize(pIndexList); + + int64_t st = taosGetTimestampUs(); + cleanupTableScanInfo(pReader->status.pTableMap); for (int32_t i = 0; i < numOfTables; ++i) { SBlockIdx* pBlockIdx = taosArrayGet(pIndexList, i); @@ -651,7 +657,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* tMapDataReset(&pScanInfo->mapData); tsdbReadBlock(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); - size += pScanInfo->mapData.nData; + sizeInDisk += pScanInfo->mapData.nData; for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { SBlock block = {0}; tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetBlock); @@ -707,7 +713,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug("load block of %d tables completed, blocks:%d in %d tables, lastBlock:%d, size:%.2f Kb, elapsed time:%.2f ms %s", - numOfTables, total, numOfQTable, pBlockNum->numOfLastBlocks, size / 1000.0, el, pReader->idStr); + numOfTables, total, numOfQTable, pBlockNum->numOfLastBlocks, sizeInDisk + / 1000.0, el, pReader->idStr); pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; @@ -1396,7 +1403,7 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf if (minKey == tsLast) { if (!init) { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } @@ -1541,7 +1548,7 @@ static int32_t doMergeThreeLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo if (minKey == tsLast) { if (!init) { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); @@ -1747,7 +1754,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, STimeWindow* pWin, - SVersionRange* pVerRange, int16_t startPos) { + SVersionRange* pVerRange, int16_t* startPos) { pLastBlockReader->uid = uid; pLastBlockReader->window = *pWin; pLastBlockReader->verRange = *pVerRange; @@ -1755,14 +1762,14 @@ static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid } static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { - if (pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { + if (*(pLastBlockReader->rowIndex) >= pLastBlockReader->lastBlockData.nRow) { return false; } - pLastBlockReader->rowIndex += 1; + *(pLastBlockReader->rowIndex) += 1; SBlockData* pBlockData = &pLastBlockReader->lastBlockData; - for(int32_t i = pLastBlockReader->rowIndex; i < pBlockData->nRow; ++i) { + for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow; ++i) { if (pBlockData->aUid[i] != pLastBlockReader->uid) { continue; } @@ -1777,23 +1784,25 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { // no data any more if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { - pLastBlockReader->rowIndex = pBlockData->nRow; + *(pLastBlockReader->rowIndex) = pBlockData->nRow; return false; } if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { - pLastBlockReader->rowIndex = pBlockData->nRow; + *(pLastBlockReader->rowIndex) = pBlockData->nRow; return false; } - pLastBlockReader->rowIndex = i; + *(pLastBlockReader->rowIndex) = i; return true; } - pLastBlockReader->rowIndex = pBlockData->nRow; + // set all data is consumed in last block + *(pLastBlockReader->rowIndex) = pBlockData->nRow; return false; } +#if 0 static int32_t saveCurrentState(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->rowIndex; } @@ -1801,15 +1810,16 @@ static int32_t saveCurrentState(SLastBlockReader* pLastBlockReader) { static void restoreState(SLastBlockReader* pLastBlockReader, int32_t state) { pLastBlockReader->rowIndex = state; } +#endif static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { SBlockData* pBlockData = &pLastBlockReader->lastBlockData; - return pBlockData->aTSKEY[pLastBlockReader->rowIndex]; + return pBlockData->aTSKEY[*pLastBlockReader->rowIndex]; } // todo handle desc order static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { - if (pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { + if (*pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { return false; } @@ -1847,7 +1857,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI STSRow* pTSRow = NULL; SRowMerger merge = {0}; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); doMergeRowsInLastBlock(pLastBlockReader, ts, &merge); @@ -1898,7 +1908,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI STSRow* pTSRow = NULL; SRowMerger merge = {0}; - TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, pLastBlockReader->rowIndex); + TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInLastBlock(pLastBlockReader, tsLastBlock, &merge); @@ -1925,12 +1935,9 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pBlockScanInfo = pReader->status.pTableIter; } - SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - - int16_t startIndex = pBlockInfo != NULL? pBlockScanInfo->indexInBlockL:-1; - initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pReader->window, &pReader->verRange, startIndex); - - bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pReader->window, &pReader->verRange, &pBlockScanInfo->indexInBlockL); +// bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; @@ -1961,7 +1968,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); // currently loaded file data block is consumed - if (pBlockData->nRow > 0 && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { + if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); setBlockAllDumped(pDumpInfo, pBlock, pReader->order); break; @@ -2252,7 +2259,7 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64 } static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; + SReaderStatus* pStatus = &pReader->status; SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; while(1) { @@ -2261,17 +2268,42 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { if (pStatus->pTableIter == NULL) { return TSDB_CODE_SUCCESS; } - } else { // let's try next table - pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); - if (pStatus->pTableIter == NULL) { - return TSDB_CODE_SUCCESS; + } + + // load the last data block of current table + // todo opt perf by avoiding load last block repeatly + STableBlockScanInfo* pScanInfo = pStatus->pTableIter; + int32_t code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); + if (code != TSDB_CODE_SUCCESS) { // todo handle error + return code; + } + + initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pReader->window, &pReader->verRange, &pScanInfo->indexInBlockL); + if (pScanInfo->indexInBlockL == -1) { + bool hasData = nextRowInLastBlock(pLastBlockReader); + if (!hasData) { // current table does not have rows in last block, try next table + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } + continue; } } - // find the last block that contain the specified block uid - return doLoadRelatedLastBlock(pLastBlockReader, pStatus->pTableIter->uid, pReader); + code = doBuildDataBlock(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - //todo check for all empty table + if (pReader->pResBlock->info.rows > 0) { + return TSDB_CODE_SUCCESS; + } + + // current table is exhausted, let's try next table + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } } } @@ -2281,8 +2313,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; - SBlock* pBlock = NULL; TSDBKEY key = {0}; + SBlock* pBlock = NULL; STableBlockScanInfo* pScanInfo = NULL; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; @@ -2292,29 +2324,28 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { pBlock = getCurrentBlock(pBlockIter); key = getCurrentKeyInBuf(pBlockIter, pReader); + // load the last data block of current table code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); if (code != TSDB_CODE_SUCCESS) { // todo handle error } - initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pReader->window, &pReader->verRange, pScanInfo->indexInBlockL); - bool hasData = nextRowInLastBlock(pLastBlockReader); - } else { - ASSERT(pBlockIter->numOfBlocks == 0); + initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pReader->window, &pReader->verRange, &pScanInfo->indexInBlockL); } - if (pBlockInfo == NULL || fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, key, pLastBlockReader)) { - if (pBlockInfo != NULL) { - tBlockDataReset(&pStatus->fileBlockData); - code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema); - if (code != TSDB_CODE_SUCCESS) { - //todo - } + if (pBlockInfo == NULL) { // build data block from last data file + ASSERT(pBlockIter->numOfBlocks == 0); + code = buildComposedDataBlock(pReader); + } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, key, pLastBlockReader)) { + tBlockDataReset(&pStatus->fileBlockData); + code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + // todo + } - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); + if (code != TSDB_CODE_SUCCESS) { + return code; } // build composed data block @@ -2425,7 +2456,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { return code; } - // all data blocks are check in last file, now let's try the next file + // all data blocks are checked in this last block file, now let's try the next file if (pReader->status.pTableIter == NULL) { code = initForFirstBlockInFile(pReader, pBlockIter); @@ -2434,7 +2465,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { return code; } - // this file does not have blocks, let's start check the last block file + // this file does not have data files, let's start check the last block file if exists if (pBlockIter->numOfBlocks == 0) { goto _begin; } @@ -2855,7 +2886,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, S while(nextRowInLastBlock(pLastBlockReader)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { - TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, pLastBlockReader->rowIndex); + TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, *pLastBlockReader->rowIndex); tRowMerge(pMerger, &fRow1); } else { break; From 98fc185e9255fa40e65633abe17ba0ef6de1795d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 18 Aug 2022 18:53:23 +0800 Subject: [PATCH 104/194] fix: escape character problem in auto create table insert --- include/libs/nodes/nodes.h | 6 +- include/util/tdef.h | 6 +- source/common/src/systable.c | 2 +- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/timewindowoperator.c | 3 +- source/libs/nodes/src/nodesToSQLFuncs.c | 122 ++++++++++++------ source/libs/parser/src/parInsert.c | 4 +- 7 files changed, 98 insertions(+), 49 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index bb75efa00a..5743d33608 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -105,7 +105,7 @@ typedef enum ENodeType { QUERY_NODE_COLUMN_REF, // Statement nodes are used in parser and planner module. - QUERY_NODE_SET_OPERATOR, + QUERY_NODE_SET_OPERATOR = 100, QUERY_NODE_SELECT_STMT, QUERY_NODE_VNODE_MODIF_STMT, QUERY_NODE_CREATE_DATABASE_STMT, @@ -198,7 +198,7 @@ typedef enum ENodeType { QUERY_NODE_QUERY, // logic plan node - QUERY_NODE_LOGIC_PLAN_SCAN, + QUERY_NODE_LOGIC_PLAN_SCAN = 1000, QUERY_NODE_LOGIC_PLAN_JOIN, QUERY_NODE_LOGIC_PLAN_AGG, QUERY_NODE_LOGIC_PLAN_PROJECT, @@ -215,7 +215,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN, // physical plan node - QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, + QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, diff --git a/include/util/tdef.h b/include/util/tdef.h index a3deb73fd4..e40d382aea 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -133,7 +133,6 @@ typedef enum EOperatorType { OP_TYPE_REM, // unary arithmetic operator OP_TYPE_MINUS, - OP_TYPE_ASSIGN, // bitwise operator OP_TYPE_BIT_AND, @@ -164,7 +163,10 @@ typedef enum EOperatorType { // json operator OP_TYPE_JSON_GET_VALUE, - OP_TYPE_JSON_CONTAINS + OP_TYPE_JSON_CONTAINS, + + // internal operator + OP_TYPE_ASSIGN } EOperatorType; #define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 6dddcc2f74..65041e1f12 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -88,7 +88,7 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "retention", .bytes = 60 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "retentions", .bytes = 60 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, {.name = "cachemodel", .bytes = TSDB_CACHE_MODEL_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "cachesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 311d82c8a2..74810d4d07 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -205,7 +205,7 @@ typedef struct SExprSupp { } SExprSupp; typedef struct SOperatorInfo { - uint8_t operatorType; + uint16_t operatorType; bool blocking; // block operator or not uint8_t status; // denote if current operator is completed char* name; // name, for debug purpose @@ -434,7 +434,7 @@ typedef struct SStreamAggSupporter { typedef struct SessionWindowSupporter { SStreamAggSupporter* pStreamAggSup; int64_t gap; - uint8_t parentType; + uint16_t parentType; SAggSupporter* pIntervalAggSup; } SessionWindowSupporter; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 6418f5305c..6101501605 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1780,7 +1780,7 @@ void increaseTs(SqlFunctionCtx* pCtx) { } } -void initIntervalDownStream(SOperatorInfo* downstream, uint8_t type, SAggSupporter* pSup) { +void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup) { if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { // Todo(liuyao) support partition by column return; @@ -2644,7 +2644,6 @@ void destroyTimeSliceOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(param); } - SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 3b129740e8..ba0232fb1a 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -21,36 +21,89 @@ #include "taoserror.h" #include "thash.h" -char *gOperatorStr[] = {NULL, - "+", - "-", - "*", - "/", - "%", - "-", - "&", - "|", - ">", - ">=", - "<", - "<=", - "=", - "<>", - "IN", - "NOT IN", - "LIKE", - "NOT LIKE", - "MATCH", - "NMATCH", - "IS NULL", - "IS NOT NULL", - "IS TRUE", - "IS FALSE", - "IS UNKNOWN", - "IS NOT TRUE", - "IS NOT FALSE", - "IS NOT UNKNOWN"}; -char *gLogicConditionStr[] = {"AND", "OR", "NOT"}; +static const char *operatorTypeStr(EOperatorType type) { + switch (type) { + case OP_TYPE_ADD: + return "+"; + case OP_TYPE_SUB: + return "-"; + case OP_TYPE_MULTI: + return "*"; + case OP_TYPE_DIV: + return "/"; + case OP_TYPE_REM: + return "%"; + case OP_TYPE_MINUS: + return "-"; + case OP_TYPE_BIT_AND: + return "&"; + case OP_TYPE_BIT_OR: + return "|"; + case OP_TYPE_GREATER_THAN: + return ">"; + case OP_TYPE_GREATER_EQUAL: + return ">="; + case OP_TYPE_LOWER_THAN: + return "<"; + case OP_TYPE_LOWER_EQUAL: + return "<="; + case OP_TYPE_EQUAL: + return "="; + case OP_TYPE_NOT_EQUAL: + return "<>"; + case OP_TYPE_IN: + return "IN"; + case OP_TYPE_NOT_IN: + return "NOT IN"; + case OP_TYPE_LIKE: + return "LIKE"; + case OP_TYPE_NOT_LIKE: + return "NOT LIKE"; + case OP_TYPE_MATCH: + return "MATCH"; + case OP_TYPE_NMATCH: + return "NMATCH"; + case OP_TYPE_IS_NULL: + return "IS NULL"; + case OP_TYPE_IS_NOT_NULL: + return "IS NOT NULL"; + case OP_TYPE_IS_TRUE: + return "IS TRUE"; + case OP_TYPE_IS_FALSE: + return "IS FALSE"; + case OP_TYPE_IS_UNKNOWN: + return "IS UNKNOWN"; + case OP_TYPE_IS_NOT_TRUE: + return "IS NOT TRUE"; + case OP_TYPE_IS_NOT_FALSE: + return "IS NOT FALSE"; + case OP_TYPE_IS_NOT_UNKNOWN: + return "IS NOT UNKNOWN"; + case OP_TYPE_JSON_GET_VALUE: + return "=>"; + case OP_TYPE_JSON_CONTAINS: + return "CONTAINS"; + case OP_TYPE_ASSIGN: + return "="; + default: + break; + } + return "UNKNOWN"; +} + +static const char *logicConditionTypeStr(ELogicConditionType type) { + switch (type) { + case LOGIC_COND_TYPE_AND: + return "AND"; + case LOGIC_COND_TYPE_OR: + return "OR"; + case LOGIC_COND_TYPE_NOT: + return "NOT"; + default: + break; + } + return "UNKNOWN"; +} int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { switch (pNode->type) { @@ -94,12 +147,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { NODES_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len)); } - if (pOpNode->opType >= (sizeof(gOperatorStr) / sizeof(gOperatorStr[0]))) { - nodesError("unknown operation type:%d", pOpNode->opType); - NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - - *len += snprintf(buf + *len, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]); + *len += snprintf(buf + *len, bufSize - *len, " %s ", operatorTypeStr(pOpNode->opType)); if (pOpNode->pRight) { NODES_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len)); @@ -118,7 +166,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { FOREACH(node, pLogicNode->pParameterList) { if (!first) { - *len += snprintf(buf + *len, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]); + *len += snprintf(buf + *len, bufSize - *len, " %s ", logicConditionTypeStr(pLogicNode->condType)); } NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 31ae35e717..0922cdb6b9 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -143,9 +143,9 @@ static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, con } char name[TSDB_DB_FNAME_LEN] = {0}; strncpy(name, pTableName->z, dbLen); - dbLen = strdequote(name); + int32_t actualDbLen = strdequote(name); - code = tNameSetDbName(pName, acctId, name, dbLen); + code = tNameSetDbName(pName, acctId, name, actualDbLen); if (code != TSDB_CODE_SUCCESS) { return buildInvalidOperationMsg(pMsgBuf, msg1); } From 4344ba56f51c1c177ce7c6c6d9ed736620ddce7d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 18 Aug 2022 19:04:26 +0800 Subject: [PATCH 105/194] add test cases --- tests/system-test/2-query/interp.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 9348a8ca8f..57f2d3b1b1 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -551,6 +551,34 @@ class TDTestCase: tdSql.checkData(0, 0, 15) tdSql.checkData(1, 0, 15) + tdLog.printNoPrefix("==========step9:test error cases") + + tdSql.query(f"select interp(c0) from tb where ts >= '2020-02-10 00:00:05' and ts <= '2020-02-15 00:00:05' range('2022-02-05 00:05:00', '2022-02-06 00:05:00') every(1d) fill(null)"); + tdSql.checkRows(2) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + + tdSql.error(f"select interp(c0) from {dbname}.{tbname}") + tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05')") + tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d)") + tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') fill(null)") + tdSql.error(f"select interp(c0) from {dbname}.{tbname} every(1s) fill(null)") + tdSql.error(f"select interp(c0) from {dbname}.{tbname} where ts >= '2020-02-10 00:00:05' and ts <= '2020-02-15 00:00:05' every(1s) fill(null)") + + # input can only be numerical types + tdSql.error(f"select interp(ts) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(c6) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(c7) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(c8) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + + # input can only be columns + tdSql.error(f"select interp(1) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(1.5) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(true) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(false) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp('abcd') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp('中文字符') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") From 5a79aa1978b2633f84c86fc3101722b2e7f6745b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 18 Aug 2022 18:48:17 +0800 Subject: [PATCH 106/194] fix(sma): memory leak --- source/dnode/mnode/impl/src/mndSma.c | 4 +++- source/dnode/mnode/impl/src/mndVgroup.c | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 6db9b59c69..2fb934aaad 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -530,7 +530,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.sourceDbUid = pDb->uid; streamObj.targetDbUid = pDb->uid; streamObj.version = 1; - streamObj.sql = pCreate->sql; + streamObj.sql = strdup(pCreate->sql); streamObj.smaId = smaObj.uid; streamObj.watermark = pCreate->watermark; streamObj.trigger = STREAM_TRIGGER_WINDOW_CLOSE; @@ -585,6 +585,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea return -1; } if (pAst != NULL) nodesDestroyNode(pAst); + nodesDestroyNode((SNode *)pPlan); int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); @@ -609,6 +610,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea code = 0; _OVER: + tFreeStreamObj(&streamObj); mndDestroySmaObj(&smaObj); mndTransDrop(pTrans); return code; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0567ec4e14..09eed7fb32 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -509,6 +509,7 @@ int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup) { pVgroup->replica = 1; if (mndGetAvailableDnode(pMnode, pDb, pVgroup, pArray) != 0) return -1; + taosArrayDestroy(pArray); mInfo("db:%s, sma vgId:%d is alloced", pDb->name, pVgroup->vgId); return 0; @@ -1862,4 +1863,4 @@ _OVER: #endif } -bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; } \ No newline at end of file +bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; } From 31872611bdab9641e6f8d6e397d449f3484be1c5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 18 Aug 2022 19:32:14 +0800 Subject: [PATCH 107/194] fix test case --- .../tsim/alter/cached_schema_after_alter.sim | 36 +++++++++---------- tests/script/tsim/alter/dnode.sim | 4 +-- tests/script/tsim/alter/table.sim | 2 +- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/script/tsim/alter/cached_schema_after_alter.sim b/tests/script/tsim/alter/cached_schema_after_alter.sim index bd2b1d272c..30b879b612 100644 --- a/tests/script/tsim/alter/cached_schema_after_alter.sim +++ b/tests/script/tsim/alter/cached_schema_after_alter.sim @@ -14,7 +14,7 @@ print ========== cached_schema_after_alter.sim sql drop database $db -x step1 step1: -sql create database $db +sql create database $db print ====== create tables sql use $db @@ -32,10 +32,10 @@ if $rows != 1 then endi if $data01 != 1 then return -1 -endi +endi if $data02 != 1 then return -1 -endi +endi sql select * from $tb2 if $rows != 1 then @@ -43,10 +43,10 @@ if $rows != 1 then endi if $data01 != 1 then return -1 -endi +endi if $data02 != 1 then return -1 -endi +endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -61,10 +61,10 @@ if $rows != 1 then endi if $data01 != 1 then return -1 -endi +endi if $data02 != 1 then return -1 -endi +endi sql select * from $tb2 print select * from $tb2 ==> $data00 $data01 $data02 @@ -73,10 +73,10 @@ if $rows != 1 then endi if $data01 != 1 then return -1 -endi +endi if $data02 != 1 then return -1 -endi +endi $ts = $ts0 + $delta sql insert into $tb2 values ( $ts , 2, 2) @@ -86,16 +86,16 @@ if $rows != 2 then endi if $data01 != 1 then return -1 -endi +endi if $data02 != 1 then return -1 -endi +endi if $data11 != 2 then return -1 -endi +endi if $data12 != 2 then return -1 -endi +endi sql select * from $tb2 order by ts asc if $rows != 2 then @@ -103,15 +103,15 @@ if $rows != 2 then endi if $data01 != 1 then return -1 -endi +endi if $data02 != 1 then return -1 -endi +endi if $data11 != 2 then return -1 -endi +endi if $data12 != 2 then return -1 -endi +endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/alter/dnode.sim b/tests/script/tsim/alter/dnode.sim index d773c1f8a9..be3c385d45 100644 --- a/tests/script/tsim/alter/dnode.sim +++ b/tests/script/tsim/alter/dnode.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -print ======== step1 +print ======== step1 sql alter dnode 1 'resetlog' sql alter dnode 1 'monitor' '1' sql alter dnode 1 'monitor' '0' @@ -65,4 +65,4 @@ sql alter dnode 1 balance "vnode:2-dnode:1" -x step4 step4: print ======= over -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/alter/table.sim b/tests/script/tsim/alter/table.sim index 48ab7ddab0..dccfc7f5d6 100644 --- a/tests/script/tsim/alter/table.sim +++ b/tests/script/tsim/alter/table.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -print ======== step1 +print ======== step1 sql create database d1 sql use d1 sql create table tb (ts timestamp, a int) From 234b89767d3b4136fa23db5ec93bfc437bce5f7c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 18 Aug 2022 19:32:14 +0800 Subject: [PATCH 108/194] fix test case --- tests/script/tsim/bnode/basic1.sim | 46 +++++++------- tests/script/tsim/compress/commitlog.sim | 20 +++--- tests/script/tsim/compress/compress.sim | 20 +++--- tests/script/tsim/compress/compress2.sim | 20 +++--- tests/script/tsim/compress/uncompress.sim | 20 +++--- tests/script/tsim/db/alter_option.sim | 34 +++++----- tests/script/tsim/db/alter_replica_13.sim | 10 +-- tests/script/tsim/db/alter_replica_31.sim | 10 +-- tests/script/tsim/db/back_insert.sim | 10 +-- tests/script/tsim/db/basic1.sim | 24 +++---- tests/script/tsim/db/basic2.sim | 8 +-- tests/script/tsim/db/basic3.sim | 8 +-- tests/script/tsim/db/basic4.sim | 54 ++++++++-------- tests/script/tsim/db/basic5.sim | 8 +-- tests/script/tsim/db/basic6.sim | 70 ++++++++++----------- tests/script/tsim/db/commit.sim | 8 +-- tests/script/tsim/db/delete_reuse1.sim | 20 +++--- tests/script/tsim/db/delete_reuse2.sim | 26 ++++---- tests/script/tsim/db/delete_reusevnode.sim | 10 +-- tests/script/tsim/db/delete_reusevnode2.sim | 2 +- tests/script/tsim/db/delete_writing1.sim | 12 ++-- tests/script/tsim/db/delete_writing2.sim | 16 ++--- tests/script/tsim/db/dropdnodes.sim | 16 ++--- tests/script/tsim/db/keep.sim | 16 ++--- tests/script/tsim/db/len.sim | 24 +++---- tests/script/tsim/db/repeat.sim | 2 +- tests/script/tsim/db/show_create_db.sim | 8 +-- tests/script/tsim/db/show_create_table.sim | 24 +++---- tests/script/tsim/db/tables.sim | 4 +- 29 files changed, 275 insertions(+), 275 deletions(-) diff --git a/tests/script/tsim/bnode/basic1.sim b/tests/script/tsim/bnode/basic1.sim index 003d0ceb3d..0a20001636 100644 --- a/tests/script/tsim/bnode/basic1.sim +++ b/tests/script/tsim/bnode/basic1.sim @@ -7,24 +7,24 @@ sql connect print =============== select * from information_schema.ins_dnodes sql select * from information_schema.ins_dnodes; -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != 1 then return -1 endi sql select * from information_schema.ins_mnodes; -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != 1 then return -1 endi -if $data02 != leader then +if $data02 != leader then return -1 endi @@ -33,62 +33,62 @@ sql create dnode $hostname port 7200 sleep 2000 sql select * from information_schema.ins_dnodes; -if $rows != 2 then +if $rows != 2 then return -1 endi -if $data00 != 1 then +if $data00 != 1 then return -1 endi -if $data10 != 2 then +if $data10 != 2 then return -1 endi print $data02 -if $data02 != 0 then +if $data02 != 0 then return -1 endi -if $data12 != 0 then +if $data12 != 0 then return -1 endi -if $data04 != ready then +if $data04 != ready then return -1 endi -if $data14 != ready then +if $data14 != ready then return -1 endi sql select * from information_schema.ins_mnodes; -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != 1 then return -1 endi -if $data02 != leader then +if $data02 != leader then return -1 endi #print =============== create drop bnode 1 #sql create bnode on dnode 1 #sql show bnodes -#if $rows != 1 then +#if $rows != 1 then # return -1 #endi -#if $data00 != 1 then +#if $data00 != 1 then # return -1 #endi #sql_error create bnode on dnode 1 # #sql drop bnode on dnode 1 #sql show bnodes -#if $rows != 0 then +#if $rows != 0 then # return -1 #endi #sql_error drop bnode on dnode 1 @@ -96,17 +96,17 @@ endi #print =============== create drop bnode 2 #sql create bnode on dnode 2 #sql show bnodes -#if $rows != 1 then +#if $rows != 1 then # return -1 #endi -#if $data00 != 2 then +#if $data00 != 2 then # return -1 #endi #sql_error create bnode on dnode 2 # #sql drop bnode on dnode 2 #sql show bnodes -#if $rows != 0 then +#if $rows != 0 then # return -1 #endi #sql_error drop bnode on dnode 2 @@ -115,7 +115,7 @@ endi #sql create bnode on dnode 1 #sql create bnode on dnode 2 #sql show bnodes -#if $rows != 2 then +#if $rows != 2 then # return -1 #endi @@ -127,7 +127,7 @@ endi # #sleep 2000 #sql show bnodes -#if $rows != 2 then +#if $rows != 2 then # return -1 #endi diff --git a/tests/script/tsim/compress/commitlog.sim b/tests/script/tsim/compress/commitlog.sim index bc9c231a9e..38899b95ba 100644 --- a/tests/script/tsim/compress/commitlog.sim +++ b/tests/script/tsim/compress/commitlog.sim @@ -25,7 +25,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -46,7 +46,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -67,7 +67,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -83,7 +83,7 @@ $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 endi @@ -93,18 +93,18 @@ $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 endi - + $i = 2 $db = $dbPrefix . $i $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/compress/compress.sim b/tests/script/tsim/compress/compress.sim index 766f97450c..4752f1ba50 100644 --- a/tests/script/tsim/compress/compress.sim +++ b/tests/script/tsim/compress/compress.sim @@ -25,7 +25,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -47,7 +47,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -69,7 +69,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -85,7 +85,7 @@ $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 endi @@ -95,18 +95,18 @@ $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 endi - + $i = 2 $db = $dbPrefix . $i $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/compress/compress2.sim b/tests/script/tsim/compress/compress2.sim index 87e50cce5b..c55b74f246 100644 --- a/tests/script/tsim/compress/compress2.sim +++ b/tests/script/tsim/compress/compress2.sim @@ -26,7 +26,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -48,7 +48,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -70,7 +70,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -86,7 +86,7 @@ $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 endi @@ -96,18 +96,18 @@ $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 endi - + $i = 2 $db = $dbPrefix . $i $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/compress/uncompress.sim b/tests/script/tsim/compress/uncompress.sim index ccd5db4b0c..f48fc6da23 100644 --- a/tests/script/tsim/compress/uncompress.sim +++ b/tests/script/tsim/compress/uncompress.sim @@ -26,7 +26,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -48,7 +48,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -70,7 +70,7 @@ while $count < $N endw sql select * from $tb -if $rows != $N then +if $rows != $N then return -1 endi @@ -85,7 +85,7 @@ $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 endi @@ -95,18 +95,18 @@ $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 endi - + $i = 2 $db = $dbPrefix . $i $tb = $tbPrefix . $i sql use $db sql select * from $tb print select * from $tb ==> $rows points -if $rows != $N then +if $rows != $N then return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index d81671eebd..3d260235f2 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -186,13 +186,13 @@ sql_error alter database db replica 0 #sql alter database db replica 1 #sql select * from information_schema.ins_databases #print replica: $data4_db -#if $data4_db != 1 then +#if $data4_db != 1 then # return -1 #endi #sql alter database db replica 3 #sql select * from information_schema.ins_databases #print replica: $data4_db -#if $data4_db != 3 then +#if $data4_db != 3 then # return -1 #endi @@ -200,13 +200,13 @@ sql_error alter database db replica 0 #sql alter database db quorum 2 #sql select * from information_schema.ins_databases #print quorum $data5_db -#if $data5_db != 2 then +#if $data5_db != 2 then # return -1 #endi #sql alter database db quorum 1 #sql select * from information_schema.ins_databases #print quorum $data5_db -#if $data5_db != 1 then +#if $data5_db != 1 then # return -1 #endi @@ -233,7 +233,7 @@ endi #sql alter database db keep 1000,2000 #sql select * from information_schema.ins_databases #print keep $data7_db -#if $data7_db != 500,500,500 then +#if $data7_db != 500,500,500 then # return -1 #endi @@ -263,13 +263,13 @@ sql_error alter database db keep -1 #sql alter database db blocks 3 #sql select * from information_schema.ins_databases #print blocks $data9_db -#if $data9_db != 3 then +#if $data9_db != 3 then # return -1 #endi #sql alter database db blocks 11 #sql select * from information_schema.ins_databases #print blocks $data9_db -#if $data9_db != 11 then +#if $data9_db != 11 then # return -1 #endi @@ -300,13 +300,13 @@ print ============== step wal_level sql alter database db wal_level 1 sql select * from information_schema.ins_databases print wal_level $data20_db -if $data20_db != 1 then +if $data20_db != 1 then return -1 endi sql alter database db wal_level 2 sql select * from information_schema.ins_databases print wal_level $data20_db -if $data20_db != 2 then +if $data20_db != 2 then return -1 endi @@ -319,19 +319,19 @@ print ============== modify wal_fsync_period sql alter database db wal_fsync_period 2000 sql select * from information_schema.ins_databases print wal_fsync_period $data21_db -if $data21_db != 2000 then +if $data21_db != 2000 then return -1 endi sql alter database db wal_fsync_period 500 sql select * from information_schema.ins_databases print wal_fsync_period $data21_db -if $data21_db != 500 then +if $data21_db != 500 then return -1 endi sql alter database db wal_fsync_period 0 sql select * from information_schema.ins_databases print wal_fsync_period $data21_db -if $data21_db != 0 then +if $data21_db != 0 then return -1 endi sql_error alter database db wal_fsync_period 180001 @@ -351,31 +351,31 @@ print ============== modify cachelast [0, 1, 2, 3] sql alter database db cachemodel 'last_value' sql select * from information_schema.ins_databases print cachelast $data18_db -if $data18_db != last_value then +if $data18_db != last_value then return -1 endi sql alter database db cachemodel 'last_row' sql select * from information_schema.ins_databases print cachelast $data18_db -if $data18_db != last_row then +if $data18_db != last_row then return -1 endi sql alter database db cachemodel 'none' sql select * from information_schema.ins_databases print cachelast $data18_db -if $data18_db != none then +if $data18_db != none then return -1 endi sql alter database db cachemodel 'last_value' sql select * from information_schema.ins_databases print cachelast $data18_db -if $data18_db != last_value then +if $data18_db != last_value then return -1 endi sql alter database db cachemodel 'both' sql select * from information_schema.ins_databases print cachelast $data18_db -if $data18_db != both then +if $data18_db != both then return -1 endi diff --git a/tests/script/tsim/db/alter_replica_13.sim b/tests/script/tsim/db/alter_replica_13.sim index d232c9bcd3..1d06d3abb9 100644 --- a/tests/script/tsim/db/alter_replica_13.sim +++ b/tests/script/tsim/db/alter_replica_13.sim @@ -36,10 +36,10 @@ endi print =============== step2: create database sql create database db vgroups 1 sql select * from information_schema.ins_databases -if $rows != 3 then +if $rows != 3 then return -1 endi -if $data(db)[4] != 1 then +if $data(db)[4] != 1 then return -1 endi @@ -82,7 +82,7 @@ step3: return -1 endi sql select * from information_schema.ins_dnodes -print ===> rows: $rows +print ===> rows: $rows print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 print ===> $data20 $data21 $data22 $data23 $data24 $data25 @@ -115,7 +115,7 @@ step4: return -1 endi sql show db.vgroups -print ===> rows: $rows +print ===> rows: $rows print ===> $data00 $data01 $data02 $data03 $data04 $data05 if $data[0][4] != leader then goto step4 @@ -137,4 +137,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim index 17ab040520..4ab6783d07 100644 --- a/tests/script/tsim/db/alter_replica_31.sim +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -23,7 +23,7 @@ step1: return -1 endi sql select * from information_schema.ins_dnodes -print ===> rows: $rows +print ===> rows: $rows print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 print ===> $data20 $data21 $data22 $data23 $data24 $data25 @@ -47,10 +47,10 @@ endi print =============== step2: create database sql create database db vgroups 1 replica 3 sql select * from information_schema.ins_databases -if $rows != 3 then +if $rows != 3 then return -1 endi -if $data(db)[4] != 3 then +if $data(db)[4] != 3 then return -1 endi @@ -139,7 +139,7 @@ step3: return -1 endi sql show db.vgroups -print ===> rows: $rows +print ===> rows: $rows if $rows != 1 then goto step3 endi @@ -165,4 +165,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/db/back_insert.sim b/tests/script/tsim/db/back_insert.sim index e2bdb3a64b..b3f8207293 100644 --- a/tests/script/tsim/db/back_insert.sim +++ b/tests/script/tsim/db/back_insert.sim @@ -2,8 +2,8 @@ sql connect $x = 1 begin: sql reset query cache - sleep 100 - sql insert into db.tb values(now, $x ) -x begin - #print ===> insert successed $x - $x = $x + 1 -goto begin \ No newline at end of file + sleep 100 + sql insert into db.tb values(now, $x ) -x begin + #print ===> insert successed $x + $x = $x + 1 +goto begin diff --git a/tests/script/tsim/db/basic1.sim b/tests/script/tsim/db/basic1.sim index 679440590f..69eeb9347b 100644 --- a/tests/script/tsim/db/basic1.sim +++ b/tests/script/tsim/db/basic1.sim @@ -25,15 +25,15 @@ endi print =============== show vgroups1 sql use d1 sql show vgroups -if $rows != 2 then +if $rows != 2 then return -1 endi -if $data00 != 2 then +if $data00 != 2 then return -1 endi -if $data10 != 3 then +if $data10 != 3 then return -1 endi @@ -59,11 +59,11 @@ if $rows != 2 then return -1 endi -if $data00 != 4 then +if $data00 != 4 then return -1 endi -if $data10 != 5 then +if $data10 != 5 then return -1 endi @@ -73,15 +73,15 @@ if $rows != 3 then return -1 endi -if $data00 != 6 then +if $data00 != 6 then return -1 endi -if $data10 != 7 then +if $data10 != 7 then return -1 endi -if $data20 != 8 then +if $data20 != 8 then return -1 endi @@ -91,19 +91,19 @@ if $rows != 4 then return -1 endi -if $data00 != 9 then +if $data00 != 9 then return -1 endi -if $data10 != 10 then +if $data10 != 10 then return -1 endi -if $data20 != 11 then +if $data20 != 11 then return -1 endi -if $data30 != 12 then +if $data30 != 12 then return -1 endi diff --git a/tests/script/tsim/db/basic2.sim b/tests/script/tsim/db/basic2.sim index 2ba7d806af..b7ac0b5edd 100644 --- a/tests/script/tsim/db/basic2.sim +++ b/tests/script/tsim/db/basic2.sim @@ -27,7 +27,7 @@ sql create table t3 (ts timestamp, i int); sql create table t4 (ts timestamp, i int); sql select * from information_schema.ins_databases -print rows: $rows +print rows: $rows print $data00 $data01 $data02 $data03 print $data10 $data11 $data12 $data13 if $rows != 3 then @@ -47,7 +47,7 @@ endi #endi sql show tables -if $rows != 4 then +if $rows != 4 then return -1 endi @@ -64,8 +64,8 @@ if $rows != 4 then endi sql show tables -if $rows != 3 then +if $rows != 3 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/basic3.sim b/tests/script/tsim/db/basic3.sim index 30faec0494..db355213db 100644 --- a/tests/script/tsim/db/basic3.sim +++ b/tests/script/tsim/db/basic3.sim @@ -23,12 +23,12 @@ if $data22 != 2 then return -1 endi -#if $data03 != 4 then +#if $data03 != 4 then # return -1 #endi sql show d1.tables -if $rows != 4 then +if $rows != 4 then return -1 endi @@ -44,8 +44,8 @@ if $rows != 4 then endi sql show d2.tables -if $rows != 3 then +if $rows != 3 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/basic4.sim b/tests/script/tsim/db/basic4.sim index f407c6352d..7a5e0ec764 100644 --- a/tests/script/tsim/db/basic4.sim +++ b/tests/script/tsim/db/basic4.sim @@ -11,109 +11,109 @@ sql create table d1.t3 (ts timestamp, i int); sql create table d1.t4 (ts timestamp, i int); sql select * from information_schema.ins_databases -if $rows != 3 then +if $rows != 3 then return -1 endi -if $data20 != d1 then +if $data20 != d1 then return -1 endi -if $data22 != 1 then +if $data22 != 1 then return -1 endi -if $data24 != 1 then +if $data24 != 1 then return -1 endi sql show d1.tables -if $rows != 4 then +if $rows != 4 then return -1 endi sql show d1.vgroups -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != 2 then +if $data00 != 2 then return -1 endi -if $data01 != d1 then +if $data01 != d1 then return -1 endi -print =============== drop table +print =============== drop table sql drop table d1.t1 sql select * from information_schema.ins_databases -if $rows != 3 then +if $rows != 3 then return -1 endi -if $data20 != d1 then +if $data20 != d1 then return -1 endi -if $data22 != 1 then +if $data22 != 1 then return -1 endi -if $data24 != 1 then +if $data24 != 1 then return -1 endi sql show d1.tables -if $rows != 3 then +if $rows != 3 then return -1 endi sql show d1.vgroups -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != 2 then +if $data00 != 2 then return -1 endi -if $data01 != d1 then +if $data01 != d1 then return -1 endi -print =============== drop all table +print =============== drop all table sql drop table d1.t2 sql drop table d1.t3 sql drop table d1.t4 sql select * from information_schema.ins_databases -if $rows != 3 then +if $rows != 3 then return -1 endi -if $data20 != d1 then +if $data20 != d1 then return -1 endi -if $data22 != 1 then +if $data22 != 1 then return -1 endi -if $data24 != 1 then +if $data24 != 1 then return -1 endi sql show d1.tables -if $rows != 0 then +if $rows != 0 then return -1 endi sql show d1.vgroups -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != 2 then +if $data00 != 2 then return -1 endi -if $data01 != d1 then +if $data01 != d1 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/basic5.sim b/tests/script/tsim/db/basic5.sim index 9b809c35f0..933fb8cf4b 100644 --- a/tests/script/tsim/db/basic5.sim +++ b/tests/script/tsim/db/basic5.sim @@ -13,13 +13,13 @@ sql create table tb1 using st1 tags(1); sql insert into tb1 values (now, 1); sql show stables -if $rows != 1 then +if $rows != 1 then print $rows return -1 endi sql show tables -if $rows != 1 then +if $rows != 1 then return -1 endi @@ -35,12 +35,12 @@ sql use db1; sql create stable st1 (ts timestamp, f1 int) tags(t1 int) sql show stables -if $rows != 1 then +if $rows != 1 then return -1 endi sql show tables -if $rows != 0 then +if $rows != 0 then return -1 endi diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 2377a65ac0..5043574787 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -14,7 +14,7 @@ $st = $stPrefix . $i $tb = $tbPrefix . $i print =============== step1 -# quorum presicion +# quorum presicion sql create database $db vgroups 8 replica 1 duration 2 keep 10 minrows 80 maxrows 10000 wal_level 2 wal_fsync_period 1000 comp 0 cachemodel 'last_value' precision 'us' sql select * from information_schema.ins_databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -46,7 +46,7 @@ endi #if $data29 != 12 then # return -1 #endi - + print =============== step2 sql_error create database $db sql create database if not exists $db @@ -60,7 +60,7 @@ sql drop database $db sql select * from information_schema.ins_databases if $rows != 2 then return -1 -endi +endi print =============== step4 sql_error drop database $db @@ -102,22 +102,22 @@ while $i < 5 sql create table $tb using $st tags(1) sql show stables - if $rows != 1 then + if $rows != 1 then return -1 endi print $data00 $data01 $data02 $data03 - if $data00 != $st then + if $data00 != $st then return -1 endi sql show tables - if $rows != 1 then + if $rows != 1 then return -1 endi print $data00 $data01 $data02 $data03 - if $data00 != $tb then + if $data00 != $tb then return -1 endi @@ -127,8 +127,8 @@ endw print =============== step7 $i = 0 while $i < 5 - $db = $dbPrefix . $i - sql drop database $db + $db = $dbPrefix . $i + sql drop database $db $i = $i + 1 endw @@ -143,20 +143,20 @@ sql create table $st (ts timestamp, i int) tags (j int) sql create table $tb using $st tags(1) sql show stables -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != $st then +if $data00 != $st then return -1 endi sql show tables -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != $tb then +if $data00 != $tb then return -1 endi @@ -168,12 +168,12 @@ sql create database $db sql use $db sql show stables -if $rows != 0 then +if $rows != 0 then return -1 endi sql show tables -if $rows != 0 then +if $rows != 0 then return -1 endi @@ -182,20 +182,20 @@ sql create table $st (ts timestamp, i int) tags (j int) sql create table $tb using $st tags(1) sql show stables -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != $st then +if $data00 != $st then return -1 endi sql show tables -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != $tb then +if $data00 != $tb then return -1 endi @@ -207,12 +207,12 @@ sql create database $db sql use $db sql show stables -if $rows != 0 then +if $rows != 0 then return -1 endi sql show tables -if $rows != 0 then +if $rows != 0 then return -1 endi @@ -221,20 +221,20 @@ sql create table $st (ts timestamp, i int) tags (j int) sql create table $tb using $st tags(1) sql show stables -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != $st then +if $data00 != $st then return -1 endi sql show tables -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != $tb then +if $data00 != $tb then return -1 endi @@ -245,12 +245,12 @@ sql insert into $tb values (now+4a, 3) sql insert into $tb values (now+5a, 4) sql select * from $tb -if $rows != 5 then +if $rows != 5 then return -1 endi sql select * from $st -if $rows != 5 then +if $rows != 5 then return -1 endi @@ -262,12 +262,12 @@ sql create database $db sql use $db sql show stables -if $rows != 0 then +if $rows != 0 then return -1 endi sql show tables -if $rows != 0 then +if $rows != 0 then return -1 endi @@ -276,20 +276,20 @@ sql create table $st (ts timestamp, i int) tags (j int) sql create table $tb using $st tags(1) sql show stables -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != $st then +if $data00 != $st then return -1 endi sql show tables -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != $tb then +if $data00 != $tb then return -1 endi @@ -300,12 +300,12 @@ sql insert into $tb values (now+4a, 3) sql insert into $tb values (now+5a, 4) sql select * from $tb -if $rows != 5 then +if $rows != 5 then return -1 endi sql select * from $st -if $rows != 5 then +if $rows != 5 then return -1 endi diff --git a/tests/script/tsim/db/commit.sim b/tests/script/tsim/db/commit.sim index 731f2aa256..2233245034 100644 --- a/tests/script/tsim/db/commit.sim +++ b/tests/script/tsim/db/commit.sim @@ -39,9 +39,9 @@ sql create table tb (ts timestamp, i int) $x = 1 while $x < 41 $time = $x . m - sql insert into tb values (now + $time , $x ) + sql insert into tb values (now + $time , $x ) $x = $x + 1 -endw +endw sql select * from tb order by ts desc print ===> rows $rows @@ -71,7 +71,7 @@ if $data01 != 40 then return -1 endi -$oldnum = $rows +$oldnum = $rows $num = $rows + 2 print ======== step3 import old data @@ -120,4 +120,4 @@ if $data01 != 40 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/db/delete_reuse1.sim b/tests/script/tsim/db/delete_reuse1.sim index 680fe6b2ed..9dcb3c6ac1 100644 --- a/tests/script/tsim/db/delete_reuse1.sim +++ b/tests/script/tsim/db/delete_reuse1.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -print ======== step1 +print ======== step1 sql create database d1 replica 1 vgroups 1 sql create database d2 replica 1 vgroups 1 sql create database d3 replica 1 vgroups 1 @@ -47,7 +47,7 @@ step2: print ========= step3 sql reset query cache -sleep 50 +sleep 50 sql create database d1 replica 1 sql create table d1.t1 (ts timestamp, i int) @@ -65,20 +65,20 @@ while $x < 20 sql insert into d1.t1 values(now, -1) -x step4 return -1 step4: - + sql create database d1 replica 1 sql reset query cache - sleep 50 + sleep 50 sql create table d1.t1 (ts timestamp, i int) sql insert into d1.t1 values(now, $x ) sql select * from d1.t1 if $rows != 1 then return -1 endi - - $x = $x + 1 - - print ===> loop times: $x -endw -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file + $x = $x + 1 + + print ===> loop times: $x +endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/delete_reuse2.sim b/tests/script/tsim/db/delete_reuse2.sim index d181b6b780..4480b60b1b 100644 --- a/tests/script/tsim/db/delete_reuse2.sim +++ b/tests/script/tsim/db/delete_reuse2.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -print ======== step1 +print ======== step1 sql create database d1 replica 1 sql create database d2 replica 1 sql create database d3 replica 1 @@ -48,7 +48,7 @@ step2: print ========= step3 sql create database db1 replica 1 sql reset query cache -sleep 50 +sleep 50 sql create table db1.tb1 (ts timestamp, i int) sql insert into db1.tb1 values(now, 2) @@ -61,7 +61,7 @@ print ========= step4 $x = 1 while $x < 20 - $db = db . $x + $db = db . $x $tb = tb . $x sql use $db sql drop database $db @@ -69,14 +69,14 @@ while $x < 20 sql insert into $tb values(now, -1) -x step4 return -1 step4: - - $x = $x + 1 - $db = db . $x + + $x = $x + 1 + $db = db . $x $tb = tb . $x - + sql reset query cache - sleep 50 - + sleep 50 + sql create database $db replica 1 sql use $db sql create table $tb (ts timestamp, i int) @@ -85,8 +85,8 @@ while $x < 20 if $rows != 1 then return -1 endi - - print ===> loop times: $x -endw -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file + print ===> loop times: $x +endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/delete_reusevnode.sim b/tests/script/tsim/db/delete_reusevnode.sim index d194f82d08..7af5c9d39d 100644 --- a/tests/script/tsim/db/delete_reusevnode.sim +++ b/tests/script/tsim/db/delete_reusevnode.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -print ======== step1 +print ======== step1 $tbPrefix = t $i = 0 @@ -21,13 +21,13 @@ while $i < 30 print times $i $i = $i + 1 -endw +endw print ======== step2 sql select * from information_schema.ins_databases -if $rows != 2 then +if $rows != 2 then return -1 -endi +endi system sh/stop_dnodes.sh @@ -94,4 +94,4 @@ if $rows != 2 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/delete_reusevnode2.sim b/tests/script/tsim/db/delete_reusevnode2.sim index 754a6d695b..91473e5ee1 100644 --- a/tests/script/tsim/db/delete_reusevnode2.sim +++ b/tests/script/tsim/db/delete_reusevnode2.sim @@ -62,4 +62,4 @@ if $rows != 2 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/delete_writing1.sim b/tests/script/tsim/db/delete_writing1.sim index 279f8dece8..6fec09989d 100644 --- a/tests/script/tsim/db/delete_writing1.sim +++ b/tests/script/tsim/db/delete_writing1.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -sql create database db +sql create database db sql create table db.tb (ts timestamp, i int) sql insert into db.tb values(now, 1) @@ -11,18 +11,18 @@ print ======== start back run_back tsim/db/back_insert.sim sleep 1000 -print ======== step1 -$x = 1 +print ======== step1 +$x = 1 while $x < 10 print drop database times $x sql drop database if exists db - sql create database db + sql create database db sql create table db.tb (ts timestamp, i int) sleep 1000 - + $x = $x + 1 endw -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/delete_writing2.sim b/tests/script/tsim/db/delete_writing2.sim index 8eab126ae8..ad156f30eb 100644 --- a/tests/script/tsim/db/delete_writing2.sim +++ b/tests/script/tsim/db/delete_writing2.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -sql create database db +sql create database db sql create table db.tb (ts timestamp, i int) sql insert into db.tb values(now, 1) @@ -11,11 +11,11 @@ sql create database db2 sql create table db2.tb2 (ts timestamp, i int) sql insert into db2.tb2 values(now, 1) -sql create database db3 +sql create database db3 sql create table db3.tb3 (ts timestamp, i int) sql insert into db3.tb3 values(now, 1) -sql create database db4 +sql create database db4 sql create table db4.tb4 (ts timestamp, i int) sql insert into db4.tb4 values(now, 1) @@ -23,19 +23,19 @@ print ======== start back run_back tsim/db/back_insert.sim sleep 1000 -print ======== step1 -$x = 1 +print ======== step1 +$x = 1 while $x < 10 print drop database times $x sql drop database if exists db - sql create database db + sql create database db sql create table db.tb (ts timestamp, i int) sleep 1000 - + $x = $x + 1 endw -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/dropdnodes.sim b/tests/script/tsim/db/dropdnodes.sim index 8a46d5f9ce..20b4a136df 100644 --- a/tests/script/tsim/db/dropdnodes.sim +++ b/tests/script/tsim/db/dropdnodes.sim @@ -12,7 +12,7 @@ system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start sleep 2000 -sql connect +sql connect sql create dnode $hostname2 sleep 2000 @@ -61,13 +61,13 @@ sql show tables print $rows if $rows != 16 then return -1 -endi +endi sql select * from mt print $rows if $rows != 16 then return -1 -endi +endi print ========== step3 @@ -82,26 +82,26 @@ sql show tables print $rows if $rows != 8 then return -1 -endi +endi sql select * from mt print $rows if $rows != 8 then return -1 -endi +endi sql select * from db.t5 if $rows != 1 then return -1 -endi +endi sql select * from db.t13 if $rows != 1 then return -1 -endi +endi sql_error select * from db.t1 sql_error select * from db.t9 system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/db/keep.sim b/tests/script/tsim/db/keep.sim index e146a666d0..f0653c4801 100644 --- a/tests/script/tsim/db/keep.sim +++ b/tests/script/tsim/db/keep.sim @@ -14,7 +14,7 @@ while $x < 41 sql insert into tb values (now - $time , $x ) -x step2 step2: $x = $x + 1 -endw +endw sql select * from tb print ===> rows $rows last $data01 @@ -42,10 +42,10 @@ sql alter database keepdb keep 60 sql flush database keepdb sql select * from information_schema.ins_databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 -if $data22 != 2 then +if $data22 != 2 then return -1 endi -if $data27 != 86400m,86400m,86400m then +if $data27 != 86400m,86400m,86400m then return -1 endi @@ -56,7 +56,7 @@ while $x < 81 sql insert into tb values (now - $time , $x ) -x step4 step4: $x = $x + 1 -endw +endw sql select * from tb print ===> rows $rows last $data01 @@ -83,10 +83,10 @@ endi print ======== step6 alter db sql alter database keepdb keep 30 sql select * from information_schema.ins_databases -if $data22 != 2 then +if $data22 != 2 then return -1 endi -if $data27 != 43200m,43200m,43200m then +if $data27 != 43200m,43200m,43200m then return -1 endi @@ -110,7 +110,7 @@ while $x < 121 sql insert into tb values (now - $time , $x ) -x step8 step8: $x = $x + 1 -endw +endw sql select * from tb print ===> rows $rows last $data01 @@ -137,4 +137,4 @@ error3: print ======= test success system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/db/len.sim b/tests/script/tsim/db/len.sim index ae475ddf47..047dafd5f8 100644 --- a/tests/script/tsim/db/len.sim +++ b/tests/script/tsim/db/len.sim @@ -11,33 +11,33 @@ sql create database -x step1 step1: sql select * from information_schema.ins_databases -if $rows != 2 then +if $rows != 2 then return -1 endi print =============== step2 sql create database a sql select * from information_schema.ins_databases -if $rows != 3 then +if $rows != 3 then return -1 endi sql drop database a sql select * from information_schema.ins_databases -if $rows != 2 then +if $rows != 2 then return -1 endi print =============== step3 sql create database a12345678 sql select * from information_schema.ins_databases -if $rows != 3 then +if $rows != 3 then return -1 endi sql drop database a12345678 sql select * from information_schema.ins_databases -if $rows != 2 then +if $rows != 2 then return -1 endi @@ -46,15 +46,15 @@ sql create database a012345678901201234567890120123456789012a0123456789012012345 return -1 step4: sql select * from information_schema.ins_databases -if $rows != 2 then +if $rows != 2 then return -1 endi print =============== step5 -sql create database a;1 +sql create database a;1 sql drop database a sql select * from information_schema.ins_databases -if $rows != 2 then +if $rows != 2 then return -1 endi @@ -64,7 +64,7 @@ sql create database a'1 -x step6 step6: sql select * from information_schema.ins_databases -if $rows != 2 then +if $rows != 2 then return -1 endi @@ -73,7 +73,7 @@ sql create database (a) -x step7 return -1 step7: sql select * from information_schema.ins_databases -if $rows != 2 then +if $rows != 2 then return -1 endi @@ -82,8 +82,8 @@ sql create database a.1 -x step8 return -1 step8: sql select * from information_schema.ins_databases -if $rows != 2 then +if $rows != 2 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/repeat.sim b/tests/script/tsim/db/repeat.sim index 98d66244f5..b0627659d0 100644 --- a/tests/script/tsim/db/repeat.sim +++ b/tests/script/tsim/db/repeat.sim @@ -56,4 +56,4 @@ sql drop database d10 sql drop database d11 sql drop database d12 -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/show_create_db.sim b/tests/script/tsim/db/show_create_db.sim index 45007d01d6..3a51fedbff 100644 --- a/tests/script/tsim/db/show_create_db.sim +++ b/tests/script/tsim/db/show_create_db.sim @@ -7,7 +7,7 @@ print =============== step2 sql create database db sql show create database db -if $rows != 1 then +if $rows != 1 then return -1 endi @@ -15,13 +15,13 @@ print =============== step3 sql use db sql show create database db -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != db then +if $data00 != db then return -1 -endi +endi sql drop database db diff --git a/tests/script/tsim/db/show_create_table.sim b/tests/script/tsim/db/show_create_table.sim index 44fa09577e..0aeee42d21 100644 --- a/tests/script/tsim/db/show_create_table.sim +++ b/tests/script/tsim/db/show_create_table.sim @@ -11,14 +11,14 @@ sql create table t0 using meters tags(1,'ch') sql create table normalTbl(ts timestamp, zone binary(8)) sql use db -sql show create table meters -if $rows != 1 then +sql show create table meters +if $rows != 1 then return -1 endi print ===============check sub table sql show create table t0 -if $rows != 1 then +if $rows != 1 then return -1 endi if $data00 == 't0' then @@ -27,8 +27,8 @@ endi print ===============check normal table -sql show create table normalTbl -if $rows != 1 then +sql show create table normalTbl +if $rows != 1 then return -1 endi @@ -37,8 +37,8 @@ if $data00 == 'normalTbl' then endi print ===============check super table -sql show create table meters -if $rows != 1 then +sql show create table meters +if $rows != 1 then return -1 endi @@ -49,7 +49,7 @@ endi print ===============check sub table with prefix sql show create table db.t0 -if $rows != 1 then +if $rows != 1 then return -1 endi @@ -58,8 +58,8 @@ if $data00 == 't0' then endi print ===============check normal table with prefix -sql show create table db.normalTbl -if $rows != 1 then +sql show create table db.normalTbl +if $rows != 1 then return -1 endi @@ -69,8 +69,8 @@ endi print ===============check super table with prefix -sql show create table db.meters -if $rows != 1 then +sql show create table db.meters +if $rows != 1 then return -1 endi diff --git a/tests/script/tsim/db/tables.sim b/tests/script/tsim/db/tables.sim index cdee504753..273a1fd45d 100644 --- a/tests/script/tsim/db/tables.sim +++ b/tests/script/tsim/db/tables.sim @@ -8,7 +8,7 @@ sql create database db sql select * from information_schema.ins_databases print $rows $data07 -if $rows != 3 then +if $rows != 3 then return -1 endi @@ -125,4 +125,4 @@ if $data01 != 4 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 8f5fa8b98270cbb47b909797054e7842008cd404 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 18 Aug 2022 19:36:40 +0800 Subject: [PATCH 109/194] fix invalid packet --- source/libs/transport/src/transSvr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index f50711c59c..68b911f553 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -278,7 +278,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { if (pBuf->invalid) { tTrace("%s conn %p alread read invalid packet", transLabel(pTransInst), conn); destroyConn(conn, true); - break; + return; } else { if (false == uvHandleReq(conn)) break; } From 873db889026d58366f8b4868d63a93ea339dd1b4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 19:37:05 +0800 Subject: [PATCH 110/194] fix --- include/common/tdataformat.h | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 1cc226f9d7..fc25e42144 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -38,19 +38,15 @@ typedef struct STagVal STagVal; typedef struct STag STag; // bitmap -const static uint8_t BIT1_MAP[8][2] = { - {0b00000000, 0b00000001}, {0b00000001, 0b00000011}, {0b00000011, 0b00000111}, {0b00000111, 0b00001111}, - {0b00001111, 0b00011111}, {0b00011111, 0b00111111}, {0b00111111, 0b01111111}, {0b01111111, 0b11111111}, -}; - const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, - {0b00000011, 0b00000111, 0b00001011, 2}, - {0b00001111, 0b00011111, 0b00101111, 4}, - {0b00111111, 0b01111111, 0b10111111, 6}}; + {0b00000000, 0b00000100, 0b00001000, 2}, + {0b00000000, 0b00010000, 0b00100000, 4}, + {0b00000000, 0b01000000, 0b10000000, 6}}; -#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) -#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) ((p)[(i) >> 3] |= BIT1_MAP[(i)&7][v]) +#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) +#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) +#define SET_BIT1(p, i, v) \ + if (v) (p)[(i) >> 3] |= (((uint8_t)1) << ((i)&7)) #define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) #define SET_BIT2(p, i, v) ((p)[(i) >> 2] |= BIT2_MAP[(i)&3][v]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) From c7d7cb7d5f6c72519f77ef13fcdc55cd4df362e8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Aug 2022 19:40:28 +0800 Subject: [PATCH 111/194] more fix --- include/common/tdataformat.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index fc25e42144..b9ce7368a7 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -43,10 +43,9 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, {0b00000000, 0b00010000, 0b00100000, 4}, {0b00000000, 0b01000000, 0b10000000, 6}}; -#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) -#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) \ - if (v) (p)[(i) >> 3] |= (((uint8_t)1) << ((i)&7)) +#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) +#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) +#define SET_BIT1(p, i, v) ((p)[(i) >> 3] |= (((uint8_t)v) << ((i)&7))) #define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) #define SET_BIT2(p, i, v) ((p)[(i) >> 2] |= BIT2_MAP[(i)&3][v]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) From 65903a7c0d3d65be07c3137f2ee84f71d03f222e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 18 Aug 2022 18:57:49 +0800 Subject: [PATCH 112/194] fix: refact mutex locking --- source/dnode/vnode/src/tsdb/tsdbCache.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 1d2c5c3b32..bb367ff8b1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1119,7 +1119,10 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH taosMemoryFree(pRow); } + taosThreadMutexUnlock(&pTsdb->lruMutex); + *handle = NULL; + return 0; } @@ -1131,15 +1134,11 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH } taosThreadMutexUnlock(&pTsdb->lruMutex); + + h = taosLRUCacheLookup(pCache, key, keyLen); } else { taosThreadMutexUnlock(&pTsdb->lruMutex); - - *handle = h; - - return code; } - - h = taosLRUCacheLookup(pCache, key, keyLen); } *handle = h; From 4e5315f1ac4ca551fdf49aacb898b61b5cbaa0c6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 Aug 2022 20:41:25 +0800 Subject: [PATCH 113/194] fix: process remaining rows out of loop --- source/libs/executor/src/scanoperator.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e578e34b82..7d188dadc7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2220,14 +2220,25 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { doFilterResult(pInfo); blockDataCleanup(p); + numOfRows = 0; + if (pInfo->pRes->info.rows > 0) { break; - } else { - numOfRows = 0; - continue; } } } + + if (numOfRows > 0) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); + doFilterResult(pInfo); + + blockDataCleanup(p); + numOfRows = 0; + } + blockDataDestroy(p); // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found From 24bfe2e2ebaef14a8d199f213ce996371183b5fa Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 18 Aug 2022 20:51:28 +0800 Subject: [PATCH 114/194] test: valgrind case --- tests/script/tsim/stream/basic0.sim | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/stream/basic0.sim b/tests/script/tsim/stream/basic0.sim index 9a5fb8012f..6d05f69dcf 100644 --- a/tests/script/tsim/stream/basic0.sim +++ b/tests/script/tsim/stream/basic0.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start -sleep 50 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 +system sh/exec.sh -n dnode1 -s start -v sql connect print =============== create database @@ -137,4 +137,17 @@ if $data13 != 789 then return -1 endi +_OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT +print =============== check +$null= + +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content > 0 then + return -1 +endi + +if $system_content == $null then + return -1 +endi From eaef3dffafb9d991773acba0c0ac375fd19d4d92 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 22:42:16 +0800 Subject: [PATCH 115/194] fix(query): fix error in query last block. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 163 +++++++++++++++++-------- 1 file changed, 112 insertions(+), 51 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f4671a0f34..5284b6936a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -88,6 +88,7 @@ typedef struct SLastBlockReader { SBlockData lastBlockData; STimeWindow window; SVersionRange verRange; + int32_t order; uint64_t uid; int16_t* rowIndex; // row index ptr, usually from the STableBlockScanInfo->indexInBlockL } SLastBlockReader; @@ -313,11 +314,11 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap } // init file iterator -static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, int32_t order, const char* idstr) { +static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdbReader* pReader/*int32_t order, const char* idstr*/) { size_t numOfFileset = taosArrayGetSize(aDFileSet); - pIter->index = ASCENDING_TRAVERSE(order) ? -1 : numOfFileset; - pIter->order = order; + pIter->index = ASCENDING_TRAVERSE(pReader->order) ? -1 : numOfFileset; + pIter->order = pReader->order; pIter->pFileList = aDFileSet; pIter->numOfFiles = numOfFileset; @@ -325,14 +326,18 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, int32 pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); if (pIter->pLastBlockReader == NULL) { int32_t code = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), idstr); + tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr); return code; } - pIter->pLastBlockReader->pBlockL = taosArrayInit(4, sizeof(SBlockL)); + SLastBlockReader* pLReader = pIter->pLastBlockReader; + pLReader->pBlockL = taosArrayInit(4, sizeof(SBlockL)); + pLReader->order = pReader->order; + pLReader->window = pReader->window; + pLReader->verRange = pReader->verRange; } - tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, idstr); + tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); return TSDB_CODE_SUCCESS; } @@ -1284,7 +1289,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc // todo here we need to each key in the last files to identify if it is really overlapped with last block bool overlapWithlastBlock = false; - if (hasDataInLastBlock(pLastBlockReader)) { + if (/*hasDataInLastBlock(pLastBlockReader)*/taosArrayGetSize(pLastBlockReader->pBlockL) > 0) { SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); } @@ -1364,7 +1369,6 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* return pReader->pMemSchema; } -// todo handle desc static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; @@ -1392,31 +1396,57 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf minKey = key; } - // file block ---> last block -----> imem -----> mem bool init = false; - if (minKey == key) { - init = true; - tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - } - if (minKey == tsLast) { - if (!init) { + // file block ---> last block -----> imem -----> mem + if (pReader->order == TSDB_ORDER_ASC) { + if (minKey == key) { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); - } + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } - if (minKey == k.ts) { - if (!init) { + if (minKey == k.ts) { + if (!init) { + init = true; + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, pRow, pSchema); + } + doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + } else { + if (minKey == k.ts) { + init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, pRow, pSchema); + doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } - doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } + + if (minKey == key) { + if (!init) { + init = true; + tRowMergerInit(&merge, &fRow, pReader->pSchema); + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + } } tRowMergerGetRow(&merge, &pTSRow); @@ -1753,23 +1783,34 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, STimeWindow* pWin, - SVersionRange* pVerRange, int16_t* startPos) { +static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, int16_t* startPos) { pLastBlockReader->uid = uid; - pLastBlockReader->window = *pWin; - pLastBlockReader->verRange = *pVerRange; pLastBlockReader->rowIndex = startPos; + + if (*startPos == -1) { + if (ASCENDING_TRAVERSE(pLastBlockReader->order)) { + // do nothing + } else { + *startPos = pLastBlockReader->lastBlockData.nRow; + } + } +} + +#define ALL_ROWS_CHECKED_INDEX INT16_MIN +static void setAllRowsChecked(SLastBlockReader *pLastBlockReader) { + *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; } static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { - if (*(pLastBlockReader->rowIndex) >= pLastBlockReader->lastBlockData.nRow) { + int32_t step = (pLastBlockReader->order == TSDB_ORDER_ASC) ? 1 : -1; + if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { return false; } - *(pLastBlockReader->rowIndex) += 1; + *(pLastBlockReader->rowIndex) += step; SBlockData* pBlockData = &pLastBlockReader->lastBlockData; - for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow; ++i) { + for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow && i >= 0; i += step) { if (pBlockData->aUid[i] != pLastBlockReader->uid) { continue; } @@ -1784,12 +1825,12 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { // no data any more if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { - *(pLastBlockReader->rowIndex) = pBlockData->nRow; + setAllRowsChecked(pLastBlockReader); return false; } if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { - *(pLastBlockReader->rowIndex) = pBlockData->nRow; + setAllRowsChecked(pLastBlockReader); return false; } @@ -1798,7 +1839,7 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { } // set all data is consumed in last block - *(pLastBlockReader->rowIndex) = pBlockData->nRow; + setAllRowsChecked(pLastBlockReader); return false; } @@ -1817,15 +1858,14 @@ static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { return pBlockData->aTSKEY[*pLastBlockReader->rowIndex]; } -// todo handle desc order static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { - if (*pLastBlockReader->rowIndex >= pLastBlockReader->lastBlockData.nRow) { + if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { return false; } - return true; } +// todo refactor static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader *pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -1849,6 +1889,25 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI if (pBlockData->nRow > 0) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + // no last block + if (pLastBlockReader->lastBlockData.nRow == 0) { + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; + } + } + // row in last file block int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader); if (ts < key) { // save rows in last block @@ -1901,7 +1960,6 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } } } else { // only last block exists - // only last block exits SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); @@ -1936,7 +1994,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pReader->window, &pReader->verRange, &pBlockScanInfo->indexInBlockL); + initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pBlockScanInfo->indexInBlockL); // bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2274,12 +2332,12 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // todo opt perf by avoiding load last block repeatly STableBlockScanInfo* pScanInfo = pStatus->pTableIter; int32_t code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); - if (code != TSDB_CODE_SUCCESS) { // todo handle error + if (code != TSDB_CODE_SUCCESS) { return code; } - initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pReader->window, &pReader->verRange, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == -1) { + initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); + if (pScanInfo->indexInBlockL == -1 || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { bool hasData = nextRowInLastBlock(pLastBlockReader); if (!hasData) { // current table does not have rows in last block, try next table pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); @@ -2327,10 +2385,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // load the last data block of current table code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); if (code != TSDB_CODE_SUCCESS) { - // todo handle error + return code; } - initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pReader->window, &pReader->verRange, &pScanInfo->indexInBlockL); + initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pScanInfo->indexInBlockL); } if (pBlockInfo == NULL) { // build data block from last data file @@ -2340,7 +2398,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { tBlockDataReset(&pStatus->fileBlockData); code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { - // todo + return code; } code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); @@ -2456,6 +2514,10 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { return code; } + if (pReader->pResBlock->info.rows > 0) { + return TSDB_CODE_SUCCESS; + } + // all data blocks are checked in this last block file, now let's try the next file if (pReader->status.pTableIter == NULL) { code = initForFirstBlockInFile(pReader, pBlockIter); @@ -2881,7 +2943,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc return TSDB_CODE_SUCCESS; } -// todo support desc order +// todo check if the rows are dropped or not int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, SRowMerger* pMerger) { while(nextRowInLastBlock(pLastBlockReader)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); @@ -3240,7 +3302,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { SDataBlockIter* pBlockIter = &pReader->status.blockIter; - initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader->order, pReader->idStr); + initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); resetDataBlockIterator(&pReader->status.blockIter, pReader->order, pReader->status.pTableMap); // no data in files, let's try buffer in memory @@ -3261,8 +3323,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl goto _err; } - initFilesetIterator(&pPrevReader->status.fileIter, pPrevReader->pReadSnap->fs.aDFileSet, pPrevReader->order, - pPrevReader->idStr); + initFilesetIterator(&pPrevReader->status.fileIter, pPrevReader->pReadSnap->fs.aDFileSet, pPrevReader); resetDataBlockIterator(&pPrevReader->status.blockIter, pPrevReader->order, pReader->status.pTableMap); // no data in files, let's try buffer in memory @@ -3507,13 +3568,13 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { tBlockDataReset(&pStatus->fileBlockData); int32_t code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pBlockScanInfo->uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { - //todo + terrno = code; + return NULL; } code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData, 1); - terrno = code; return NULL; } @@ -3555,7 +3616,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); tsdbDataFReaderClose(&pReader->pFileReader); - initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader->order, pReader->idStr); + initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); resetDataBlockIterator(&pReader->status.blockIter, pReader->order, pReader->status.pTableMap); resetDataBlockScanInfo(pReader->status.pTableMap); From 5eae03fcab5533d1a0e9b1cfe42dca9f328f27c0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Aug 2022 22:57:26 +0800 Subject: [PATCH 116/194] fix(query): handle the reverse case. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 40 +++++++++++++++++++------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5284b6936a..db943645e4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1376,24 +1376,43 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf SBlockData* pBlockData = &pReader->status.fileBlockData; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t tsLast = INT64_MIN; + if (pLastBlockReader->lastBlockData.nRow > 0) { + tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + } TSDBKEY k = TSDBROW_KEY(pRow); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t minKey = INT64_MAX; - if (minKey > tsLast) { - minKey = tsLast; - } + int64_t minKey = 0; + if (pReader->order == TSDB_ORDER_ASC) { + minKey = INT64_MAX; // chosen the minimum value + if (minKey > tsLast && pLastBlockReader->lastBlockData.nRow > 0) { + minKey = tsLast; + } - if (minKey > k.ts) { - minKey = k.ts; - } + if (minKey > k.ts) { + minKey = k.ts; + } - if (minKey > key && pBlockData->nRow > 0) { - minKey = key; + if (minKey > key && pBlockData->nRow > 0) { + minKey = key; + } + } else { + minKey = INT64_MIN; + if (minKey < tsLast && pLastBlockReader->lastBlockData.nRow > 0) { + minKey = tsLast; + } + + if (minKey < k.ts) { + minKey = k.ts; + } + + if (minKey < key && pBlockData->nRow > 0) { + minKey = key; + } } bool init = false; @@ -1530,6 +1549,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } +// todo handle the desc order check static int32_t doMergeThreeLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; From ff22c97d9964790ac3aa34b7b6a3356edd0644aa Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 19 Aug 2022 08:39:19 +0800 Subject: [PATCH 117/194] fix: user tags scan filter produces zero row will not end the sys table scan operator --- source/libs/executor/src/scanoperator.c | 34 ++++++++++++++++++------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7d188dadc7..5689b70824 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2038,10 +2038,34 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { metaReaderClear(&smr); if (numOfRows >= pOperator->resultInfo.capacity) { - break; + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); + doFilterResult(pInfo); + + blockDataCleanup(p); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } } } + if (numOfRows > 0) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); + doFilterResult(pInfo); + + blockDataCleanup(p); + numOfRows = 0; + } + + blockDataDestroy(p); + // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found if (ret != 0) { metaCloseTbCursor(pInfo->pCur); @@ -2049,14 +2073,6 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); } - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); - doFilterResult(pInfo); - - blockDataDestroy(p); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } From fc0745825e4cb7b30e1a12f118bbf8d80c9750b8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 09:51:40 +0800 Subject: [PATCH 118/194] more snapshot --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 132 +++++++-------------- 1 file changed, 40 insertions(+), 92 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 29a64cf0b5..dbf710bb20 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -28,13 +28,15 @@ struct STsdbSnapReader { SDataFReader* pDataFReader; SArray* aBlockIdx; // SArray SArray* aBlockL; // SArray - int32_t iBlockIdx; SBlockIdx* pBlockIdx; - int32_t iBlockL; - SMapData mBlock; // SMapData - int32_t iBlock; - SBlockData oBlockData; - SBlockData nBlockData; + SBlockL* pBlockL; + + int32_t iBlockIdx; + int32_t iBlockL; + SMapData mBlock; // SMapData + int32_t iBlock; + SBlockData oBlockData; + SBlockData nBlockData; // for del file int8_t delDone; SDelFReader* pDelFReader; @@ -75,96 +77,42 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { } while (true) { -#if 0 - if (pReader->pBlockIdx == NULL) { - if (pReader->iBlockIdx >= taosArrayGetSize(pReader->aBlockIdx)) { - tsdbDataFReaderClose(&pReader->pDataFReader); - break; + if (pReader->pBlockIdx && pReader->pBlockL) { + } else if (pReader->pBlockIdx) { + // may have problem (todo) + while (pReader->iBlock < pReader->mBlock.nItem) { + SBlock block; + tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetBlock); + + if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) { + // load data (todo) + } + + // next + pReader->iBlock++; + + if (*ppData) goto _exit; } + } else if (pReader->pBlockL) { + while (pReader->pBlockL) { + if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { + // load data (todo) + } - pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); - pReader->iBlockIdx++; + // next + pReader->iBlockL++; + if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) { + pReader->pBlockL = (SBlockL*)taosArrayGetSize(pReader->aBlockL); + } else { + pReader->pBlockL = NULL; + } - code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); - if (code) goto _err; - - pReader->iBlock = 0; + if (*ppData) goto _exit; + } + } else { + tsdbDataFReaderClose(&pReader->pDataFReader); + break; } - - SBlock block; - SBlock* pBlock = █ - while (true) { - if (pReader->iBlock >= pReader->mBlock.nItem) { - pReader->pBlockIdx = NULL; - break; - } - - tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, pBlock, tGetBlock); - pReader->iBlock++; - - if (pBlock->minVersion > pReader->ever || pBlock->maxVersion < pReader->sver) continue; - - code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->oBlockData, NULL, NULL); - if (code) goto _err; - - // filter - tBlockDataReset(&pReader->nBlockData); - for (int32_t iColData = 0; iColData < taosArrayGetSize(pReader->oBlockData.aIdx); iColData++) { - SColData* pColDataO = tBlockDataGetColDataByIdx(&pReader->oBlockData, iColData); - SColData* pColDataN = NULL; - - code = tBlockDataAddColData(&pReader->nBlockData, taosArrayGetSize(pReader->nBlockData.aIdx), &pColDataN); - if (code) goto _err; - - tColDataInit(pColDataN, pColDataO->cid, pColDataO->type, pColDataO->smaOn); - } - - for (int32_t iRow = 0; iRow < pReader->oBlockData.nRow; iRow++) { - TSDBROW row = tsdbRowFromBlockData(&pReader->oBlockData, iRow); - int64_t version = TSDBROW_VERSION(&row); - - tsdbTrace("vgId:%d, vnode snapshot tsdb read for %s, %" PRId64 "(%" PRId64 " , %" PRId64 ")", - TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path, version, pReader->sver, pReader->ever); - - if (version < pReader->sver || version > pReader->ever) continue; - - code = tBlockDataAppendRow(&pReader->nBlockData, &row, NULL); - if (code) goto _err; - } - - if (pReader->nBlockData.nRow <= 0) { - continue; - } - - // org data - // compress data (todo) - int32_t size = sizeof(TABLEID) + tPutBlockData(NULL, &pReader->nBlockData); - - *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); - if (*ppData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); - pHdr->type = pReader->type; - pHdr->size = size; - - TABLEID* pId = (TABLEID*)(&pHdr[1]); - pId->suid = pReader->pBlockIdx->suid; - pId->uid = pReader->pBlockIdx->uid; - - tPutBlockData((uint8_t*)(&pId[1]), &pReader->nBlockData); - - // tsdbInfo("vgId:%d, vnode snapshot read data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64 - // " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d size:%d", - // TD_VID(pTsdb->pVnode), pTsdb->path, pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid, - // pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow, - // size); - - goto _exit; - } -#endif } } From 823fbf9f777e62855c9ba0db0645e2536cba72c9 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 19 Aug 2022 10:05:55 +0800 Subject: [PATCH 119/194] fix test case --- tests/system-test/2-query/interp.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 57f2d3b1b1..934ba9e161 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -553,11 +553,6 @@ class TDTestCase: tdLog.printNoPrefix("==========step9:test error cases") - tdSql.query(f"select interp(c0) from tb where ts >= '2020-02-10 00:00:05' and ts <= '2020-02-15 00:00:05' range('2022-02-05 00:05:00', '2022-02-06 00:05:00') every(1d) fill(null)"); - tdSql.checkRows(2) - tdSql.checkData(0, 0, None) - tdSql.checkData(1, 0, None) - tdSql.error(f"select interp(c0) from {dbname}.{tbname}") tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05')") tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d)") From 06c8d14f99c075b4e08743a1d00c87f3c05fc4d7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 19 Aug 2022 10:25:16 +0800 Subject: [PATCH 120/194] refactor rpc code --- source/libs/transport/src/transSvr.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index db05aefe7b..447db76136 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -275,16 +275,15 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { if (pBuf->len <= TRANS_PACKET_LIMIT) { while (transReadComplete(pBuf)) { tTrace("%s conn %p alread read complete packet", transLabel(pTransInst), conn); - if (pBuf->invalid) { - tTrace("%s conn %p alread read invalid packet", transLabel(pTransInst), conn); + if (true == pBuf->invalid || false == uvHandleReq(conn)) { + tError("%s conn %p read invalid packet", transLabel(pTransInst), conn); destroyConn(conn, true); return; - } else { - if (false == uvHandleReq(conn)) break; } } return; } else { + tError("%s conn %p read invalid packet, exceed limit", transLabel(pTransInst), conn); destroyConn(conn, true); return; } From f59bc9c0a75931155e2262c2f495e35418494983 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 11:02:33 +0800 Subject: [PATCH 121/194] more work --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 33 ++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index dbf710bb20..a970b0be74 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -30,6 +30,8 @@ struct STsdbSnapReader { SArray* aBlockL; // SArray SBlockIdx* pBlockIdx; SBlockL* pBlockL; + SBlock* pBlock; + SBlock block; int32_t iBlockIdx; int32_t iBlockL; @@ -58,7 +60,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->fid = pSet->fid; // load - code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); + code = tsdbDataFReaderOpen(&pReader->pDataFReader, pTsdb, pSet); if (code) goto _err; code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); @@ -69,8 +71,25 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { // init pReader->iBlockIdx = 0; - pReader->pBlockIdx = NULL; + if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) { + pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); + + code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); + if (code) goto _err; + + pReader->iBlock = 0; + if (pReader->iBlock < pReader->mBlock.nItem) { + tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &pReader->block, tGetBlock); + } + } else { + pReader->pBlockIdx = NULL; + } pReader->iBlockL = 0; + if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) { + pReader->pBlockL = (SBlockL*)taosArrayGet(pReader->aBlockL, pReader->iBlockL); + } else { + pReader->pBlockL = NULL; + } tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path, pReader->fid); @@ -78,6 +97,16 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { while (true) { if (pReader->pBlockIdx && pReader->pBlockL) { + TABLEID minId = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; + TABLEID maxId = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->maxUid}; + + if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) { + // TODO + } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) { + // TODO + } else { + // TODO + } } else if (pReader->pBlockIdx) { // may have problem (todo) while (pReader->iBlock < pReader->mBlock.nItem) { From 09cb575300a720f5c2e470885d4dffc15af825e0 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 19 Aug 2022 11:10:56 +0800 Subject: [PATCH 122/194] fix: escape character problem in auto create table insert --- include/util/tdef.h | 12 ++++++------ source/libs/executor/src/executor.c | 16 ++++++++-------- source/libs/executor/src/timewindowoperator.c | 2 +- source/libs/nodes/src/nodesCodeFuncs.c | 1 - 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index e40d382aea..f3ab9ecffa 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -132,14 +132,14 @@ typedef enum EOperatorType { OP_TYPE_DIV, OP_TYPE_REM, // unary arithmetic operator - OP_TYPE_MINUS, + OP_TYPE_MINUS = 20, // bitwise operator - OP_TYPE_BIT_AND, + OP_TYPE_BIT_AND = 30, OP_TYPE_BIT_OR, // binary comparison operator - OP_TYPE_GREATER_THAN, + OP_TYPE_GREATER_THAN = 40, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, @@ -152,7 +152,7 @@ typedef enum EOperatorType { OP_TYPE_MATCH, OP_TYPE_NMATCH, // unary comparison operator - OP_TYPE_IS_NULL, + OP_TYPE_IS_NULL = 100, OP_TYPE_IS_NOT_NULL, OP_TYPE_IS_TRUE, OP_TYPE_IS_FALSE, @@ -162,11 +162,11 @@ typedef enum EOperatorType { OP_TYPE_IS_NOT_UNKNOWN, // json operator - OP_TYPE_JSON_GET_VALUE, + OP_TYPE_JSON_GET_VALUE = 150, OP_TYPE_JSON_CONTAINS, // internal operator - OP_TYPE_ASSIGN + OP_TYPE_ASSIGN = 250 } EOperatorType; #define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 4c3d5cf7af..d8f63cb008 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -348,7 +348,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, taosThreadOnce(&initPoolOnce, initRefPool); atexit(cleanupRefPool); - qDebug("start to create subplan task, TID:0x%"PRIx64 " QID:0x%"PRIx64, taskId, pSubplan->id.queryId); + qDebug("start to create subplan task, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId); int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); if (code != TSDB_CODE_SUCCESS) { @@ -374,7 +374,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } } - qDebug("subplan task create completed, TID:0x%"PRIx64 " QID:0x%"PRIx64, taskId, pSubplan->id.queryId); + qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId); _error: // if failed to add ref for all tables in this query, abort current query @@ -427,7 +427,7 @@ int waitMoment(SQInfo* pQInfo) { #endif static void freeBlock(void* param) { - SSDataBlock* pBlock = *(SSDataBlock**) param; + SSDataBlock* pBlock = *(SSDataBlock**)param; blockDataDestroy(pBlock); } @@ -467,12 +467,12 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); - int32_t current = 0; + int32_t current = 0; SSDataBlock* pRes = NULL; int64_t st = taosGetTimestampUs(); - while((pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot)) != NULL) { + while ((pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot)) != NULL) { SSDataBlock* p = createOneDataBlock(pRes, true); current += p->info.rows; ASSERT(p->info.rows > 0); @@ -494,7 +494,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; qDebug("%s task suspended, %d rows in %d blocks returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", - GET_TASKID(pTaskInfo), current, (int32_t) taosArrayGetSize(pResList), total, 0, el / 1000.0); + GET_TASKID(pTaskInfo), current, (int32_t)taosArrayGetSize(pResList), total, 0, el / 1000.0); atomic_store_64(&pTaskInfo->owner, 0); return pTaskInfo->code; @@ -632,7 +632,7 @@ int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { SOperatorInfo* pOperator = pTaskInfo->pRoot; while (1) { - uint8_t type = pOperator->operatorType; + uint16_t type = pOperator->operatorType; if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { *scanner = pOperator->info; return 0; @@ -691,7 +691,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { pTaskInfo->streamInfo.prepareStatus = *pOffset; if (!tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) { while (1) { - uint8_t type = pOperator->operatorType; + uint16_t type = pOperator->operatorType; pOperator->status = OP_OPENED; // TODO add more check if (type != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 6101501605..ed68fd2964 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3541,7 +3541,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num } void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark, - uint8_t type) { + uint16_t type) { ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type}; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index a6546f3299..0f32001c47 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -4673,7 +4673,6 @@ static int32_t jsonToNode(const SJson* pJson, void* pObj) { int32_t code; tjsonGetNumberValue(pJson, jkNodeType, pNode->type, code); - ; if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, nodesNodeName(pNode->type), jsonToSpecificNode, pNode); if (TSDB_CODE_SUCCESS != code) { From 69173f769482f5a1cb853ec057a83e77298458cd Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang <59301069+xiao-yu-wang@users.noreply.github.com> Date: Fri, 19 Aug 2022 11:15:02 +0800 Subject: [PATCH 123/194] Update index.mdx --- docs/zh/07-develop/02-model/index.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/07-develop/02-model/index.mdx b/docs/zh/07-develop/02-model/index.mdx index be545e8813..1609eb5362 100644 --- a/docs/zh/07-develop/02-model/index.mdx +++ b/docs/zh/07-develop/02-model/index.mdx @@ -11,10 +11,10 @@ TDengine 采用类关系型数据模型，需要建库、建表。因此对于 不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为了在各种场景下 TDengine 都能最大效率的工作，TDengine 建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除 SQL 标准的选项外，还可以指定保留时长、副本数、缓存大小、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如： ```sql -CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 VGROUPS 100 WAL 1; +CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 WAL_LEVEL 1; ``` -上述语句将创建一个名为 power 的库，这个库的数据将保留 365 天（超过 365 天将被自动删除），每 10 天一个数据文件，每个 VNODE 的写入内存池的大小为 16 MB，数据库的 VGROUPS 数量，对该数据库入会写 WAL 但不执行 FSYNC。详细的语法及参数请见 [数据库管理](/taos-sql/database) 章节。 +上述语句将创建一个名为 power 的库，这个库的数据将保留 365 天（超过 365 天将被自动删除），每 10 天一个数据文件，每个 VNODE 的写入内存池的大小为 16 MB，对该数据库入会写 WAL 但不执行 FSYNC。详细的语法及参数请见 [数据库管理](/taos-sql/database) 章节。 创建库之后，需要使用 SQL 命令 `USE` 将当前库切换过来，例如： From 692fdf3397482b77c4a356ca7e74ff8eeff63515 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 19 Aug 2022 11:31:52 +0800 Subject: [PATCH 124/194] enh(stream): optimize output --- source/libs/executor/src/timewindowoperator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 02be01cdb8..60822ce770 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -910,11 +910,11 @@ int32_t compareWinRes(void* pKey, void* data, int32_t index) { } static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { - if (!pUpdatedMap || taosHashGetSize(pUpdatedMap) == 0) { + int32_t delSize = taosArrayGetSize(pDelWins); + if (taosHashGetSize(pUpdatedMap) == 0 || delSize == 0) { return; } - int32_t delSize = taosArrayGetSize(pDelWins); - void* pIte = NULL; + void* pIte = NULL; while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { SResKeyPos* pResKey = (SResKeyPos*)pIte; int32_t index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes); From 98012440d44414bd5b2549ce4ddbafcac35ac169 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 19 Aug 2022 11:37:00 +0800 Subject: [PATCH 125/194] fix possible deadlock --- source/libs/scheduler/src/schRemote.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 8c9003a9b2..ecd9daf1bc 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -20,7 +20,7 @@ #include "tmsg.h" #include "tref.h" #include "trpc.h" - +// clang-format off int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { int32_t lastMsgType = pTask->lastMsgType; int32_t taskStatus = SCH_GET_TASK_STATUS(pTask); @@ -402,7 +402,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); if (pMsg) { - taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pData); } return TSDB_CODE_SUCCESS; } @@ -415,7 +415,7 @@ int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) { if (head->isHbParam) { taosMemoryFree(pMsg->pData); - + SSchHbCallbackParam *hbParam = (SSchHbCallbackParam *)param; SSchTrans trans = {.pTrans = hbParam->pTrans, .pHandle = NULL}; SCH_ERR_RET(schUpdateHbConnection(&hbParam->nodeEpId, &trans)); @@ -1104,7 +1104,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, #if 1 SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; - schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); + code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); msg = NULL; SCH_ERR_JRET(code); @@ -1114,7 +1114,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, #else if (TDMT_VND_SUBMIT != msgType) { SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; - schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); + code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); msg = NULL; SCH_ERR_JRET(code); @@ -1136,3 +1136,4 @@ _return: taosMemoryFreeClear(msg); SCH_RET(code); } +// clang-format on From cd6c9def671c8cb639baefd922ca7eed9a5b1937 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Fri, 19 Aug 2022 11:38:41 +0800 Subject: [PATCH 126/194] feat(taosAdapter): taosAdapter supports Windows (#16229) --- cmake/taosadapter_CMakeLists.txt.in | 2 +- packaging/tools/make_install.bat | 10 ++++++++ tools/CMakeLists.txt | 37 ++++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index ed8216be91..f182beed33 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 3d21433 + GIT_TAG abed566 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index 3c27c1beca..d4dde391c8 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -28,6 +28,13 @@ if not exist %tagert_dir%\\driver ( if not exist C:\\TDengine\\cfg\\taos.cfg ( copy %source_dir%\\packaging\\cfg\\taos.cfg %tagert_dir%\\cfg\\taos.cfg > nul ) + +if exist %binary_dir%\\test\\cfg\\taosadapter.toml ( + if not exist %tagert_dir%\\cfg\\taosadapter.toml ( + copy %binary_dir%\\test\\cfg\\taosadapter.toml %tagert_dir%\\cfg\\taosadapter.toml > nul + ) +) + copy %source_dir%\\include\\client\\taos.h %tagert_dir%\\include > nul copy %source_dir%\\include\\util\\taoserror.h %tagert_dir%\\include > nul copy %source_dir%\\include\\libs\\function\\taosudf.h %tagert_dir%\\include > nul @@ -40,6 +47,9 @@ copy %binary_dir%\\build\\bin\\udfd.exe %tagert_dir% > nul if exist %binary_dir%\\build\\bin\\taosBenchmark.exe ( copy %binary_dir%\\build\\bin\\taosBenchmark.exe %tagert_dir% > nul ) +if exist %binary_dir%\\build\\bin\\taosadapter.exe ( + copy %binary_dir%\\build\\bin\\taosadapter.exe %tagert_dir% > nul +) mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)&& echo To start/stop TDengine with administrator privileges: sc start/stop taosd &goto :eof :hasAdmin diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 25d6e33175..a79cbbe77d 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -128,6 +128,7 @@ ELSE () COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin ) + unset(_upx_prefix) ELSEIF (TD_DARWIN) include(ExternalProject) ExternalProject_Add(taosadapter @@ -149,8 +150,42 @@ ELSE () COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin ) +# unset(_upx_prefix) + ELSEIF (TD_WINDOWS) + include(ExternalProject) + set(_upx_prefix "${CMAKE_BINARY_DIR}/.taos/externals/upx") + ExternalProject_Add(upx + PREFIX "${_upx_prefix}" + URL https://github.com/upx/upx/releases/download/v3.96/upx-3.96-win32.zip + CONFIGURE_COMMAND cmake -E true + BUILD_COMMAND cmake -E true + INSTALL_COMMAND cmake -E true + ) + + ExternalProject_Add(taosadapter + PREFIX "taosadapter" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosadapter + BUILD_ALWAYS off + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosadapter no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND set CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client + COMMAND set CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib + COMMAND go build -ldflags "-s -w -X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" + COMMAND go build -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" + INSTALL_COMMAND + COMMAND ${_upx_prefix}/src/upx/upx taosadapter.exe + COMMAND cmake -E copy taosadapter.exe ${CMAKE_BINARY_DIR}/build/bin + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin + ) unset(_upx_prefix) ELSE () - MESSAGE("${Yellow} Windows system still use original embedded httpd ${ColourReset}") + MESSAGE("${Yellow} taosAdapter Not supported yet ${ColourReset}") ENDIF () ENDIF () From e5da9167ffc5de9cd82bde709715edd5fe4cb7bb Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 19 Aug 2022 11:52:30 +0800 Subject: [PATCH 127/194] fix: escape character problem in auto create table insert --- include/libs/nodes/querynodes.h | 3 ++ include/util/tdef.h | 2 +- source/libs/nodes/src/nodesToSQLFuncs.c | 4 +- source/libs/scalar/inc/filterInt.h | 1 - source/libs/scalar/src/filter.c | 46 ++----------------- .../libs/scalar/test/scalar/scalarTests.cpp | 6 +-- 6 files changed, 12 insertions(+), 50 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 088da73a1a..e1f86bae58 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -428,6 +428,9 @@ void nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal); char* nodesGetFillModeString(EFillMode mode); int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc); +const char* operatorTypeStr(EOperatorType type); +const char* logicConditionTypeStr(ELogicConditionType type); + #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index f3ab9ecffa..6ce1571656 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -166,7 +166,7 @@ typedef enum EOperatorType { OP_TYPE_JSON_CONTAINS, // internal operator - OP_TYPE_ASSIGN = 250 + OP_TYPE_ASSIGN = 200 } EOperatorType; #define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index ba0232fb1a..e521c57c3d 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -21,7 +21,7 @@ #include "taoserror.h" #include "thash.h" -static const char *operatorTypeStr(EOperatorType type) { +const char *operatorTypeStr(EOperatorType type) { switch (type) { case OP_TYPE_ADD: return "+"; @@ -91,7 +91,7 @@ static const char *operatorTypeStr(EOperatorType type) { return "UNKNOWN"; } -static const char *logicConditionTypeStr(ELogicConditionType type) { +const char *logicConditionTypeStr(ELogicConditionType type) { switch (type) { case LOGIC_COND_TYPE_AND: return "AND"; diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 54e873065b..23693c785a 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -350,7 +350,6 @@ struct SFilterInfo { extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); -extern OptrStr gOptrStr[]; #ifdef __cplusplus } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 1664a4d612..4377dbf14e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -24,46 +24,6 @@ #include "ttime.h" #include "functionMgt.h" -OptrStr gOptrStr[] = { - {0, "invalid"}, - {OP_TYPE_ADD, "+"}, - {OP_TYPE_SUB, "-"}, - {OP_TYPE_MULTI, "*"}, - {OP_TYPE_DIV, "/"}, - {OP_TYPE_REM, "%"}, - {OP_TYPE_MINUS, "minus"}, - {OP_TYPE_ASSIGN, "assign"}, - // bit operator - {OP_TYPE_BIT_AND, "&"}, - {OP_TYPE_BIT_OR, "|"}, - - // comparison operator - {OP_TYPE_GREATER_THAN, ">"}, - {OP_TYPE_GREATER_EQUAL, ">="}, - {OP_TYPE_LOWER_THAN, "<"}, - {OP_TYPE_LOWER_EQUAL, "<="}, - {OP_TYPE_EQUAL, "=="}, - {OP_TYPE_NOT_EQUAL, "!="}, - {OP_TYPE_IN, "in"}, - {OP_TYPE_NOT_IN, "not in"}, - {OP_TYPE_LIKE, "like"}, - {OP_TYPE_NOT_LIKE, "not like"}, - {OP_TYPE_MATCH, "match"}, - {OP_TYPE_NMATCH, "nmatch"}, - {OP_TYPE_IS_NULL, "is null"}, - {OP_TYPE_IS_NOT_NULL, "not null"}, - {OP_TYPE_IS_TRUE, "is true"}, - {OP_TYPE_IS_FALSE, "is false"}, - {OP_TYPE_IS_UNKNOWN, "is unknown"}, - {OP_TYPE_IS_NOT_TRUE, "not true"}, - {OP_TYPE_IS_NOT_FALSE, "not false"}, - {OP_TYPE_IS_NOT_UNKNOWN, "not unknown"}, - - // json operator - {OP_TYPE_JSON_GET_VALUE, "->"}, - {OP_TYPE_JSON_CONTAINS, "json contains"} -}; - bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(maxv, minr); return result >= 0; @@ -986,7 +946,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi } else { int32_t paramNum = scalarGetOperatorParamNum(optr); if (1 != paramNum) { - fltError("invalid right field in unit, operator:%s, rightType:%d", gOptrStr[optr].str, u->right.type); + fltError("invalid right field in unit, operator:%s, rightType:%d", operatorTypeStr(optr), u->right.type); return TSDB_CODE_QRY_APP_ERROR; } } @@ -1517,7 +1477,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SColumnNode *refNode = (SColumnNode *)left->desc; if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){ - len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->dataBlockId, refNode->slotId, gOptrStr[unit->compare.optr].str); + len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->dataBlockId, refNode->slotId, operatorTypeStr(unit->compare.optr)); } if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { @@ -1536,7 +1496,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); if (unit->compare.optr2 >= 0 && unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS){ - sprintf(str + strlen(str), "[%d][%d] %s [", refNode->dataBlockId, refNode->slotId, gOptrStr[unit->compare.optr2].str); + sprintf(str + strlen(str), "[%d][%d] %s [", refNode->dataBlockId, refNode->slotId, operatorTypeStr(unit->compare.optr2)); } if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 9b40f0a465..7229fdec38 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1089,16 +1089,16 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do }else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV || opType == OP_TYPE_REM || opType == OP_TYPE_MINUS){ - printf("op:%s,1result:%f,except:%f\n", gOptrStr[opType].str, *((double *)colDataGetData(column, 0)), exceptValue); + printf("op:%s,1result:%f,except:%f\n", operatorTypeStr(opType), *((double *)colDataGetData(column, 0)), exceptValue); ASSERT_TRUE(fabs(*((double *)colDataGetData(column, 0)) - exceptValue) < 0.0001); }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){ - printf("op:%s,2result:%" PRId64 ",except:%f\n", gOptrStr[opType].str, *((int64_t *)colDataGetData(column, 0)), exceptValue); + printf("op:%s,2result:%" PRId64 ",except:%f\n", operatorTypeStr(opType), *((int64_t *)colDataGetData(column, 0)), exceptValue); ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue); }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN || opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL || opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE || opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){ - printf("op:%s,3result:%d,except:%f\n", gOptrStr[opType].str, *((bool *)colDataGetData(column, 0)), exceptValue); + printf("op:%s,3result:%d,except:%f\n", operatorTypeStr(opType), *((bool *)colDataGetData(column, 0)), exceptValue); ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue); } From c7158f782ebfdf4771699fd6501fcb24e12d6271 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 11:52:43 +0800 Subject: [PATCH 128/194] adjust more --- source/dnode/vnode/src/inc/meta.h | 1 - source/dnode/vnode/src/meta/metaQuery.c | 4 ++- source/dnode/vnode/src/tsdb/tsdbFile.c | 15 ++++++-- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 42 +++++++++++++--------- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 2efc33a8ee..adfbb91920 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -66,7 +66,6 @@ int32_t metaCacheOpen(SMeta* pMeta); void metaCacheClose(SMeta* pMeta); int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo); int32_t metaCacheDrop(SMeta* pMeta, int64_t uid); -int32_t metaCacheGet(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo); struct SMeta { TdThreadRwlock lock; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 9c879bf369..9bbff6c6bd 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -127,7 +127,7 @@ _err: // return 0; // } -bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { +bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { // query uid.idx if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { return false; @@ -1051,6 +1051,8 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj return TSDB_CODE_SUCCESS; } +int32_t metaCacheGet(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo); + int32_t metaGetInfo(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo) { int32_t code = 0; void *pData = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 5db02fa285..00d2ac848f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -188,11 +188,16 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { n += tPutI32v(p ? p + n : p, pSet->diskId.level); n += tPutI32v(p ? p + n : p, pSet->diskId.id); n += tPutI32v(p ? p + n : p, pSet->fid); + + // data n += tPutHeadFile(p ? p + n : p, pSet->pHeadF); n += tPutDataFile(p ? p + n : p, pSet->pDataF); - n += tPutLastFile(p ? p + n : p, pSet->pLastF); n += tPutSmaFile(p ? p + n : p, pSet->pSmaF); + // last + n += tPutU8(p ? p + n : p, 1); // for future compatibility + n += tPutLastFile(p ? p + n : p, pSet->pLastF); + return n; } @@ -202,11 +207,17 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { n += tGetI32v(p + n, &pSet->diskId.level); n += tGetI32v(p + n, &pSet->diskId.id); n += tGetI32v(p + n, &pSet->fid); + + // data n += tGetHeadFile(p + n, pSet->pHeadF); n += tGetDataFile(p + n, pSet->pDataF); - n += tGetLastFile(p + n, pSet->pLastF); n += tGetSmaFile(p + n, pSet->pSmaF); + // last + uint8_t nLast; + n += tGetU8(p + n, &nLast); + n += tGetLastFile(p + n, pSet->pLastF); + return n; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a970b0be74..744f16b113 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -30,8 +30,6 @@ struct STsdbSnapReader { SArray* aBlockL; // SArray SBlockIdx* pBlockIdx; SBlockL* pBlockL; - SBlock* pBlock; - SBlock block; int32_t iBlockIdx; int32_t iBlockL; @@ -78,9 +76,6 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { if (code) goto _err; pReader->iBlock = 0; - if (pReader->iBlock < pReader->mBlock.nItem) { - tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &pReader->block, tGetBlock); - } } else { pReader->pBlockIdx = NULL; } @@ -97,18 +92,16 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { while (true) { if (pReader->pBlockIdx && pReader->pBlockL) { - TABLEID minId = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; - TABLEID maxId = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->maxUid}; + TABLEID id = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; - if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) { - // TODO - } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) { - // TODO - } else { - // TODO - } + // if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) { + // // TODO + // } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) { + // // TODO + // } else { + // // TODO + // } } else if (pReader->pBlockIdx) { - // may have problem (todo) while (pReader->iBlock < pReader->mBlock.nItem) { SBlock block; tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetBlock); @@ -119,9 +112,24 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { // next pReader->iBlock++; - - if (*ppData) goto _exit; + if (*ppData) break; } + + if (pReader->iBlock >= pReader->mBlock.nItem) { + pReader->iBlockIdx++; + if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) { + pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); + + code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); + if (code) goto _err; + + pReader->iBlock = 0; + } else { + pReader->pBlockIdx = NULL; + } + } + + if (*ppData) goto _exit; } else if (pReader->pBlockL) { while (pReader->pBlockL) { if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { From e25600e8e8035a581be2a9ffaf22ee063ce83f14 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 19 Aug 2022 11:57:32 +0800 Subject: [PATCH 129/194] fix crash_gen error --- source/libs/transport/src/transCli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 763483cbf6..7052b0b915 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1432,7 +1432,7 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran if (pThrd == NULL && valid == false) { transFreeMsg(pReq->pCont); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - return -1; + return TSDB_CODE_RPC_BROKEN_LINK; } TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); @@ -1477,7 +1477,7 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs if (pThrd == NULL && valid == false) { transFreeMsg(pReq->pCont); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - return -1; + return TSDB_CODE_RPC_BROKEN_LINK; } tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); From f20887ee97bc6a6daf2a74d0e39cda1ab17e0789 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 19 Aug 2022 12:28:27 +0800 Subject: [PATCH 130/194] fix(query): fix interp + fill(linear) datapoint outside interpolation window will be included TD-18392 --- source/libs/executor/src/timewindowoperator.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 02be01cdb8..c870d9d939 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2413,11 +2413,11 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } + } - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; } if (ts == pSliceInfo->current) { From a642607c68a10a5812bccf7c2b9f45a0021ba2c2 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 19 Aug 2022 14:09:33 +0800 Subject: [PATCH 131/194] os: fix flashing black console error --- contrib/CMakeLists.txt | 8 +++-- source/util/src/tconfig.c | 68 ++++++++++++++------------------------- 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 494d1577fc..2dc7622f46 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -331,9 +331,11 @@ endif(${BUILD_WITH_TRAFT}) # LIBUV if(${BUILD_WITH_UV}) - if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") - MESSAGE("Windows need set no-sign-compare") - add_compile_options(-Wno-sign-compare) + if (TD_WINDOWS) + # There is no GetHostNameW function on win7. + file(READ "libuv/src/win/util.c" LIBUV_WIN_UTIL_CONTENT) + string(REPLACE "if (GetHostNameW(buf, UV_MAXHOSTNAMESIZE" "DWORD nSize = UV_MAXHOSTNAMESIZE;\n if (GetComputerNameW(buf, &nSize" LIBUV_WIN_UTIL_CONTENT "${LIBUV_WIN_UTIL_CONTENT}") + file(WRITE "libuv/src/win/util.c" "${LIBUV_WIN_UTIL_CONTENT}") endif () add_subdirectory(libuv EXCLUDE_FROM_ALL) endif(${BUILD_WITH_UV}) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index fdb397561d..84e74b6612 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -33,6 +33,8 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd); int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); +extern char **environ; + SConfig *cfgInit() { SConfig *pCfg = taosMemoryCalloc(1, sizeof(SConfig)); if (pCfg == NULL) { @@ -627,24 +629,17 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { } int32_t cfgLoadFromEnvVar(SConfig *pConfig) { - char *line = NULL, *name, *value, *value2, *value3; + char line[1024], *name, *value, *value2, *value3; int32_t olen, vlen, vlen2, vlen3; int32_t code = 0; - ssize_t _bytes = 0; - TdCmdPtr pCmd = taosOpenCmd("set"); - if (pCmd == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - while (!taosEOFCmd(pCmd)) { + char **pEnv = environ; + line[1023] = 0; + while(*pEnv != NULL) { name = value = value2 = value3 = NULL; olen = vlen = vlen2 = vlen3 = 0; - _bytes = taosGetLineCmd(pCmd, &line); - if (_bytes < 0) { - break; - } - if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; + strncpy(line, *pEnv, sizeof(line)-1); + *pEnv++; taosEnvToCfg(line, line); paGetToken(line, &name, &olen); @@ -671,9 +666,6 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { } } - taosCloseCmd(&pCmd); - if (line != NULL) taosMemoryFreeClear(line); - uInfo("load from env variables cfg success"); return 0; } @@ -1040,34 +1032,25 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl index++; } - char *line = NULL; - ssize_t _bytes = 0; - TdCmdPtr pCmd = taosOpenCmd("set"); - if (pCmd != NULL) { - while (!taosEOFCmd(pCmd)) { - _bytes = taosGetLineCmd(pCmd, &line); - if (_bytes < 0) { - break; - } - if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { - char *p = strchr(line, '='); - if (p != NULL) { + char line[1024]; + char **pEnv = environ; + line[1023] = 0; + while(*pEnv != NULL) { + strncpy(line, *pEnv, sizeof(line)-1); + *pEnv++; + if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { + char *p = strchr(line, '='); + if (p != NULL) { + p++; + if (*p == '\'') { p++; - if (*p == '\'') { - p++; - p[strlen(p)-1] = '\0'; - } - memcpy(apolloUrl, p, TMIN(strlen(p)+1,PATH_MAX)); - uInfo("get apollo url from env variables success, apolloUrl=%s",apolloUrl); - taosCloseCmd(&pCmd); - if (line != NULL) taosMemoryFreeClear(line); - return 0; + p[strlen(p)-1] = '\0'; } + memcpy(apolloUrl, p, TMIN(strlen(p)+1,PATH_MAX)); + uInfo("get apollo url from env variables success, apolloUrl=%s",apolloUrl); + return 0; } } - taosCloseCmd(&pCmd); - if (line != NULL) taosMemoryFreeClear(line); } const char *filepath = ".env"; @@ -1083,10 +1066,11 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl return 0; } } + int64_t _bytes; TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); if (pFile != NULL) { while (!taosEOFFile(pFile)) { - _bytes = taosGetLineFile(pFile, &line); + _bytes = taosGetsFile(pFile, sizeof(line) - 1, &line); if (_bytes <= 0) { break; } @@ -1101,14 +1085,12 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl } memcpy(apolloUrl, p, TMIN(strlen(p)+1,PATH_MAX)); taosCloseFile(&pFile); - if (line != NULL) taosMemoryFreeClear(line); uInfo("get apollo url from env file success"); return 0; } } } taosCloseFile(&pFile); - if (line != NULL) taosMemoryFreeClear(line); } uInfo("fail get apollo url from cmd env file"); From 301ac18a3d4432cacd45b43624f554bd6f8d5d15 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 19 Aug 2022 14:21:47 +0800 Subject: [PATCH 132/194] os: fix flashing black console error --- source/util/src/tconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 84e74b6612..32f7b23a47 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -1070,7 +1070,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); if (pFile != NULL) { while (!taosEOFFile(pFile)) { - _bytes = taosGetsFile(pFile, sizeof(line) - 1, &line); + _bytes = taosGetsFile(pFile, sizeof(line) - 1, line); if (_bytes <= 0) { break; } From 390c2aa36056cc3373b14c221f22da50edc9a851 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 19 Aug 2022 14:26:08 +0800 Subject: [PATCH 133/194] fix: escape character problem in auto create table insert --- tests/system-test/1-insert/time_range_wise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 8f61da221d..c31d8d2547 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -293,7 +293,7 @@ class TDTestCase: dbname = tdSql.getData(0,0) tdSql.query("select * from information_schema.ins_databases") for index , value in enumerate(tdSql.cursor.description): - if value[0] == "retention": + if value[0] == "retentions": r_index = index break for row in tdSql.queryResult: From 72471af0ce2c8d110c0e47684e5de89e8cd68ab8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 14:26:37 +0800 Subject: [PATCH 134/194] adjust more code --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 87 ++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index ad759f1ef8..70d723cb3d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1850,6 +1850,8 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol uint8_t **ppBuf) { int32_t code = 0; + ASSERT(pColData->flag && (pColData->flag != HAS_NONE) && (pColData->flag != HAS_NULL)); + pBlockCol->szBitmap = 0; pBlockCol->szOffset = 0; pBlockCol->szValue = 0; @@ -1857,9 +1859,47 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol int32_t size = 0; // bitmap if (pColData->flag != HAS_VALUE) { - code = tsdbCmprData(pColData->pBitMap, BIT2_SIZE(pColData->nVal), TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, - nOut + size, &pBlockCol->szBitmap, ppBuf); + uint8_t *pBitMap = pColData->pBitMap; + int32_t szBitMap = BIT2_SIZE(pColData->nVal); + + // BIT2 to BIT1 + if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { + szBitMap = BIT1_SIZE(pColData->nVal); + pBitMap = taosMemoryCalloc(1, szBitMap); + if (pBitMap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + uint8_t v = GET_BIT2(pColData->pBitMap, iVal); + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + SET_BIT1(pBitMap, iVal, v); + break; + case (HAS_VALUE | HAS_NONE): + if (v) { + SET_BIT1(pBitMap, iVal, 1); + } else { + SET_BIT1(pBitMap, iVal, 0); + } + break; + case (HAS_VALUE | HAS_NULL): + SET_BIT1(pBitMap, iVal, v - 1); + break; + default: + ASSERT(0); + } + } + } + + code = tsdbCmprData(pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size, &pBlockCol->szBitmap, + ppBuf); if (code) goto _exit; + + if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { + taosMemoryFree(pBitMap); + } } size += pBlockCol->szBitmap; @@ -1909,9 +1949,46 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in uint8_t *p = pIn; // bitmap if (pBlockCol->szBitmap) { - code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, - BIT2_SIZE(pColData->nVal), ppBuf); - if (code) goto _exit; + if (pBlockCol->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { + uint8_t *pBitMap = NULL; + code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pBitMap, + BIT1_SIZE(pColData->nVal), ppBuf); + if (code) goto _exit; + + code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal)); + if (code) { + tFree(pBitMap); + goto _exit; + } + + // BIT1 to BIT2 + for (int32_t iVal = 0; iVal < nVal; iVal++) { + uint8_t v = GET_BIT1(pBitMap, iVal); + switch (pBlockCol->flag) { + case (HAS_NULL | HAS_NONE): + SET_BIT2(pColData->pBitMap, iVal, v); + break; + case (HAS_VALUE | HAS_NONE): + if (v) { + SET_BIT2(pColData->pBitMap, iVal, 2); + } else { + SET_BIT2(pColData->pBitMap, iVal, 0); + } + break; + case (HAS_VALUE | HAS_NULL): + SET_BIT2(pColData->pBitMap, iVal, v + 1); + break; + default: + ASSERT(0); + } + } + + tFree(pBitMap); + } else { + code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, + BIT2_SIZE(pColData->nVal), ppBuf); + if (code) goto _exit; + } } p += pBlockCol->szBitmap; From 721593ec8f8d1501c11dbc724f67a8665f00275d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 19 Aug 2022 15:03:55 +0800 Subject: [PATCH 135/194] docs: query data doc adjust --- docs/en/07-develop/02-model/index.mdx | 2 +- docs/en/07-develop/04-query-data/index.mdx | 49 +++++++++++----------- docs/zh/07-develop/04-query-data/index.mdx | 49 +++++++++++----------- 3 files changed, 51 insertions(+), 49 deletions(-) diff --git a/docs/en/07-develop/02-model/index.mdx b/docs/en/07-develop/02-model/index.mdx index b647c845d0..06a5346141 100644 --- a/docs/en/07-develop/02-model/index.mdx +++ b/docs/en/07-develop/02-model/index.mdx @@ -9,7 +9,7 @@ The data model employed by TDengine is similar to that of a relational database. The [characteristics of time-series data](https://www.taosdata.com/blog/2019/07/09/86.html) from different data collection points may be different. Characteristics include collection frequency, retention policy and others which determine how you create and configure the database. For e.g. days to keep, number of replicas, data block size, whether data updates are allowed and other configurable parameters would be determined by the characteristics of your data and your business requirements. For TDengine to operate with the best performance, we strongly recommend that you create and configure different databases for data with different characteristics. This allows you, for example, to set up different storage and retention policies. When creating a database, there are a lot of parameters that can be configured such as, the days to keep data, the number of replicas, the number of memory blocks, time precision, the minimum and maximum number of rows in each data block, whether compression is enabled, the time range of the data in single data file and so on. Below is an example of the SQL statement to create a database. ```sql -CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 VGROUPS 100 WAL 1; +CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 WAL_LEVEL 1; ``` In the above SQL statement: diff --git a/docs/en/07-develop/04-query-data/index.mdx b/docs/en/07-develop/04-query-data/index.mdx index a212fa9529..1dfcecc359 100644 --- a/docs/en/07-develop/04-query-data/index.mdx +++ b/docs/en/07-develop/04-query-data/index.mdx @@ -61,20 +61,20 @@ In summary, records across subtables can be aggregated by a simple query on thei In TDengine CLI `taos`, use the SQL below to get the average voltage of all the meters in California grouped by location. ``` -taos> SELECT AVG(voltage) FROM meters GROUP BY location; - avg(voltage) | location | -============================================================= - 222.000000000 | California.LosAngeles | - 219.200000000 | California.SanFrancisco | -Query OK, 2 row(s) in set (0.002136s) +taos> SELECT AVG(voltage), location FROM meters GROUP BY location; + avg(voltage) | location | +=============================================================================================== + 219.200000000 | California.SanFrancisco | + 221.666666667 | California.LosAngeles | +Query OK, 2 rows in database (0.005995s) ``` ### Example 2 -In TDengine CLI `taos`, use the SQL below to get the number of rows and the maximum current in the past 24 hours from meters whose groupId is 2. +In TDengine CLI `taos`, use the SQL below to get the number of rows and the maximum current from meters whose groupId is 2. ``` -taos> SELECT count(*), max(current) FROM meters where groupId = 2 and ts > now - 24h; +taos> SELECT count(*), max(current) FROM meters where groupId = 2; count(*) | max(current) | ================================== 5 | 13.4 | @@ -88,40 +88,41 @@ Join queries are only allowed between subtables of the same STable. In [Select]( In IoT use cases, down sampling is widely used to aggregate data by time range. The `INTERVAL` keyword in TDengine can be used to simplify the query by time window. For example, the SQL statement below can be used to get the sum of current every 10 seconds from meters table d1001. ``` -taos> SELECT sum(current) FROM d1001 INTERVAL(10s); - ts | sum(current) | +taos> SELECT _wstart, sum(current) FROM d1001 INTERVAL(10s); + _wstart | sum(current) | ====================================================== 2018-10-03 14:38:00.000 | 10.300000191 | 2018-10-03 14:38:10.000 | 24.900000572 | -Query OK, 2 row(s) in set (0.000883s) +Query OK, 2 rows in database (0.003139s) ``` Down sampling can also be used for STable. For example, the below SQL statement can be used to get the sum of current from all meters in California. ``` -taos> SELECT SUM(current) FROM meters where location like "California%" INTERVAL(1s); - ts | sum(current) | +taos> SELECT _wstart, SUM(current) FROM meters where location like "California%" INTERVAL(1s); + _wstart | sum(current) | ====================================================== 2018-10-03 14:38:04.000 | 10.199999809 | - 2018-10-03 14:38:05.000 | 32.900000572 | + 2018-10-03 14:38:05.000 | 23.699999809 | 2018-10-03 14:38:06.000 | 11.500000000 | 2018-10-03 14:38:15.000 | 12.600000381 | - 2018-10-03 14:38:16.000 | 36.000000000 | -Query OK, 5 row(s) in set (0.001538s) + 2018-10-03 14:38:16.000 | 34.400000572 | +Query OK, 5 rows in database (0.007413s) ``` Down sampling also supports time offset. For example, the below SQL statement can be used to get the sum of current from all meters but each time window must start at the boundary of 500 milliseconds. ``` -taos> SELECT SUM(current) FROM meters INTERVAL(1s, 500a); - ts | sum(current) | +taos> SELECT _wstart, SUM(current) FROM meters INTERVAL(1s, 500a); + _wstart | sum(current) | ====================================================== - 2018-10-03 14:38:04.500 | 11.189999809 | - 2018-10-03 14:38:05.500 | 31.900000572 | - 2018-10-03 14:38:06.500 | 11.600000000 | - 2018-10-03 14:38:15.500 | 12.300000381 | - 2018-10-03 14:38:16.500 | 35.000000000 | -Query OK, 5 row(s) in set (0.001521s) + 2018-10-03 14:38:03.500 | 10.199999809 | + 2018-10-03 14:38:04.500 | 10.300000191 | + 2018-10-03 14:38:05.500 | 13.399999619 | + 2018-10-03 14:38:06.500 | 11.500000000 | + 2018-10-03 14:38:14.500 | 12.600000381 | + 2018-10-03 14:38:16.500 | 34.400000572 | +Query OK, 6 rows in database (0.005515s) ``` In many use cases, it's hard to align the timestamp of the data collected by each collection point. However, a lot of algorithms like FFT require the data to be aligned with same time interval and application programs have to handle this by themselves. In TDengine, it's easy to achieve the alignment using down sampling. diff --git a/docs/zh/07-develop/04-query-data/index.mdx b/docs/zh/07-develop/04-query-data/index.mdx index 68f49d9f2b..2631d147a5 100644 --- a/docs/zh/07-develop/04-query-data/index.mdx +++ b/docs/zh/07-develop/04-query-data/index.mdx @@ -54,20 +54,20 @@ Query OK, 2 row(s) in set (0.001100s) 在 TAOS Shell，查找加利福尼亚州所有智能电表采集的电压平均值，并按照 location 分组。 ``` -taos> SELECT AVG(voltage) FROM meters GROUP BY location; - avg(voltage) | location | -============================================================= - 222.000000000 | California.LosAngeles | - 219.200000000 | California.SanFrancisco | -Query OK, 2 row(s) in set (0.002136s) +taos> SELECT AVG(voltage), location FROM meters GROUP BY location; + avg(voltage) | location | +=============================================================================================== + 219.200000000 | California.SanFrancisco | + 221.666666667 | California.LosAngeles | +Query OK, 2 rows in database (0.005995s) ``` ### 示例二 -在 TAOS shell, 查找 groupId 为 2 的所有智能电表过去 24 小时的记录条数，电流的最大值。 +在 TAOS shell, 查找 groupId 为 2 的所有智能电表的记录条数，电流的最大值。 ``` -taos> SELECT count(*), max(current) FROM meters where groupId = 2 and ts > now - 24h; +taos> SELECT count(*), max(current) FROM meters where groupId = 2; cunt(*) | max(current) | ================================== 5 | 13.4 | @@ -81,40 +81,41 @@ Query OK, 1 row(s) in set (0.002136s) 物联网场景里，经常需要通过降采样（down sampling）将采集的数据按时间段进行聚合。TDengine 提供了一个简便的关键词 interval 让按照时间窗口的查询操作变得极为简单。比如，将智能电表 d1001 采集的电流值每 10 秒钟求和 ``` -taos> SELECT sum(current) FROM d1001 INTERVAL(10s); - ts | sum(current) | +taos> SELECT _wstart, sum(current) FROM d1001 INTERVAL(10s); + _wstart | sum(current) | ====================================================== 2018-10-03 14:38:00.000 | 10.300000191 | 2018-10-03 14:38:10.000 | 24.900000572 | -Query OK, 2 row(s) in set (0.000883s) +Query OK, 2 rows in database (0.003139s) ``` 降采样操作也适用于超级表，比如：将加利福尼亚州所有智能电表采集的电流值每秒钟求和 ``` -taos> SELECT SUM(current) FROM meters where location like "California%" INTERVAL(1s); - ts | sum(current) | +taos> SELECT _wstart, SUM(current) FROM meters where location like "California%" INTERVAL(1s); + _wstart | sum(current) | ====================================================== 2018-10-03 14:38:04.000 | 10.199999809 | - 2018-10-03 14:38:05.000 | 32.900000572 | + 2018-10-03 14:38:05.000 | 23.699999809 | 2018-10-03 14:38:06.000 | 11.500000000 | 2018-10-03 14:38:15.000 | 12.600000381 | - 2018-10-03 14:38:16.000 | 36.000000000 | -Query OK, 5 row(s) in set (0.001538s) + 2018-10-03 14:38:16.000 | 34.400000572 | +Query OK, 5 rows in database (0.007413s) ``` 降采样操作也支持时间偏移，比如：将所有智能电表采集的电流值每秒钟求和，但要求每个时间窗口从 500 毫秒开始 ``` -taos> SELECT SUM(current) FROM meters INTERVAL(1s, 500a); - ts | sum(current) | +taos> SELECT _wstart, SUM(current) FROM meters INTERVAL(1s, 500a); + _wstart | sum(current) | ====================================================== - 2018-10-03 14:38:04.500 | 11.189999809 | - 2018-10-03 14:38:05.500 | 31.900000572 | - 2018-10-03 14:38:06.500 | 11.600000000 | - 2018-10-03 14:38:15.500 | 12.300000381 | - 2018-10-03 14:38:16.500 | 35.000000000 | -Query OK, 5 row(s) in set (0.001521s) + 2018-10-03 14:38:03.500 | 10.199999809 | + 2018-10-03 14:38:04.500 | 10.300000191 | + 2018-10-03 14:38:05.500 | 13.399999619 | + 2018-10-03 14:38:06.500 | 11.500000000 | + 2018-10-03 14:38:14.500 | 12.600000381 | + 2018-10-03 14:38:16.500 | 34.400000572 | +Query OK, 6 rows in database (0.005515s) ``` 物联网场景里，每个数据采集点采集数据的时间是难同步的，但很多分析算法(比如 FFT)需要把采集的数据严格按照时间等间隔的对齐，在很多系统里，需要应用自己写程序来处理，但使用 TDengine 的降采样操作就轻松解决。 From 44919ef147af047155eff447b5d211c74fafa771 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 19 Aug 2022 15:03:57 +0800 Subject: [PATCH 136/194] os: fix flashing black console error --- source/util/src/tconfig.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 32f7b23a47..eb45f216f4 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -639,7 +639,7 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { olen = vlen = vlen2 = vlen3 = 0; strncpy(line, *pEnv, sizeof(line)-1); - *pEnv++; + (*pEnv)++; taosEnvToCfg(line, line); paGetToken(line, &name, &olen); @@ -1037,7 +1037,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl line[1023] = 0; while(*pEnv != NULL) { strncpy(line, *pEnv, sizeof(line)-1); - *pEnv++; + (*pEnv)++; if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(line, '='); if (p != NULL) { From 189469b71161159df846bdae00543c926c4cfeaf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 15:30:54 +0800 Subject: [PATCH 137/194] fix more --- include/common/tdataformat.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index b9ce7368a7..af7c88acde 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -43,11 +43,12 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, {0b00000000, 0b00010000, 0b00100000, 4}, {0b00000000, 0b01000000, 0b10000000, 6}}; +#define N1(n) ((((uint8_t)1) << (n)) - 1) #define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) #define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) ((p)[(i) >> 3] |= (((uint8_t)v) << ((i)&7))) +#define SET_BIT1(p, i, v) ((p)[(i) >> 3] = (p)[(i) >> 3] & N1((i)&7) | (((uint8_t)(v)) << ((i)&7))) #define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) ((p)[(i) >> 2] |= BIT2_MAP[(i)&3][v]) +#define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) // STSchema From 3dbc5d8abce0059f5ed8345ce34d68cc93ce3667 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 19 Aug 2022 16:20:54 +0800 Subject: [PATCH 138/194] fix(query): fix "n","y" time unit parsing does not return error TD-18462 --- source/common/src/ttime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 77b45b6df1..60145f3ad4 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -679,7 +679,7 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati *unit = token[tokenLen - 1]; if (*unit == 'n' || *unit == 'y') { - return 0; + return -1; } return getDuration(*duration, *unit, duration, timePrecision); From 0010cf3b1d0b7ca04d3e975e4bcae51f3d1ae2bf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 19 Aug 2022 16:21:59 +0800 Subject: [PATCH 139/194] fix http report --- source/libs/transport/src/thttp.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 4e49e9ca13..7cfb188ac9 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -21,11 +21,14 @@ #include "taoserror.h" #include "tlog.h" + +#define HTTP_RECV_BUF_SIZE 1024 typedef struct SHttpClient { uv_connect_t conn; uv_tcp_t tcp; uv_write_t req; - uv_buf_t* buf; + uv_buf_t* wbuf; + char *rbuf; char* addr; uint16_t port; } SHttpClient; @@ -122,14 +125,30 @@ _OVER: } static void destroyHttpClient(SHttpClient* cli) { - taosMemoryFree(cli->buf); + taosMemoryFree(cli->wbuf); + taosMemoryFree(cli->rbuf); taosMemoryFree(cli->addr); taosMemoryFree(cli); + } static void clientCloseCb(uv_handle_t* handle) { SHttpClient* cli = handle->data; destroyHttpClient(cli); } +static void clientAllocBuffCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { + SHttpClient* cli = handle->data; + buf->base = cli->rbuf; + buf->len = HTTP_RECV_BUF_SIZE; +} +static void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t *buf) { + SHttpClient* cli = handle->data; + if (nread < 0) { + uError("http-report read error:%s", uv_err_name(nread)); + } else { + uInfo("http-report succ to read %d bytes, just ignore it", nread); + } + uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); +} static void clientSentCb(uv_write_t* req, int32_t status) { SHttpClient* cli = req->data; if (status != 0) { @@ -138,7 +157,7 @@ static void clientSentCb(uv_write_t* req, int32_t status) { } else { uInfo("http-report succ to send data"); } - uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); + uv_read_start((uv_stream_t *)&cli->tcp, clientAllocBuffCb, clientRecvCb); } static void clientConnCb(uv_connect_t* req, int32_t status) { SHttpClient* cli = req->data; @@ -148,7 +167,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); return; } - uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->buf, 2, clientSentCb); + uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb); } static int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { @@ -189,7 +208,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 cli->conn.data = cli; cli->tcp.data = cli; cli->req.data = cli; - cli->buf = wb; + cli->wbuf = wb; + cli->rbuf = taosMemoryCalloc(1, HTTP_RECV_BUF_SIZE); cli->addr = tstrdup(server); cli->port = port; @@ -199,7 +219,6 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 int32_t fd = taosCreateSocketWithTimeout(5); uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); - int32_t ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&dest, clientConnCb); if (ret != 0) { uError("http-report failed to connect to server, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); From ad2708d3708c25c9c887539efe9310bd9e9a241f Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 19 Aug 2022 16:48:34 +0800 Subject: [PATCH 140/194] os: fix flashing black console error --- source/util/src/tconfig.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index eb45f216f4..2a28ec66d2 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -639,7 +639,7 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { olen = vlen = vlen2 = vlen3 = 0; strncpy(line, *pEnv, sizeof(line)-1); - (*pEnv)++; + pEnv++; taosEnvToCfg(line, line); paGetToken(line, &name, &olen); @@ -1037,7 +1037,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl line[1023] = 0; while(*pEnv != NULL) { strncpy(line, *pEnv, sizeof(line)-1); - (*pEnv)++; + pEnv++; if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(line, '='); if (p != NULL) { From 206a1c3de99eaccc6696c0c84f5f59d5c6e6683e Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 19 Aug 2022 16:50:44 +0800 Subject: [PATCH 141/194] Revert "fix(query): fix "n","y" time unit parsing does not return error" This reverts commit 3dbc5d8abce0059f5ed8345ce34d68cc93ce3667. --- source/common/src/ttime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 60145f3ad4..77b45b6df1 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -679,7 +679,7 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati *unit = token[tokenLen - 1]; if (*unit == 'n' || *unit == 'y') { - return -1; + return 0; } return getDuration(*duration, *unit, duration, timePrecision); From 029c16e487216fce3d3ea9fa21f6a19980449e95 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 17:13:39 +0800 Subject: [PATCH 142/194] more code --- source/dnode/vnode/src/inc/tsdb.h | 8 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 - source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 2 + source/dnode/vnode/src/tsdb/tsdbUtil.c | 140 +++++++++++++----- 4 files changed, 106 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0c706c3b6b..9e02453ccf 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -65,6 +65,7 @@ typedef struct SBlockInfo SBlockInfo; typedef struct SSmaInfo SSmaInfo; typedef struct SBlockCol SBlockCol; +#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 @@ -150,12 +151,7 @@ SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); - -#if 1 -int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); -int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); -int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); -#endif +int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 55ef576916..dc9cf19593 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -15,8 +15,6 @@ #include "tsdb.h" -#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) - // SDelFWriter ==================================================== int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 744f16b113..ce970ea870 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -94,6 +94,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { if (pReader->pBlockIdx && pReader->pBlockL) { TABLEID id = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; + ASSERT(0); + // if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) { // // TODO // } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 70d723cb3d..740da45393 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1500,50 +1500,114 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD *ppColData = NULL; } -int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData) { - int32_t n = 0; +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut) { + int32_t code = 0; - n += tPutI32v(p ? p + n : p, pBlockData->nRow); - if (p) { - memcpy(p + n, pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow); - } - n = n + sizeof(int64_t) * pBlockData->nRow; - if (p) { - memcpy(p + n, pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow); - } - n = n + sizeof(TSKEY) * pBlockData->nRow; + SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, + .fmtVer = 0, + .suid = pBlockData->suid, + .uid = pBlockData->uid, + .nRow = pBlockData->nRow, + .cmprAlg = cmprAlg}; - int32_t nCol = taosArrayGetSize(pBlockData->aIdx); - n += tPutI32v(p ? p + n : p, nCol); - for (int32_t iCol = 0; iCol < nCol; iCol++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); - n += tPutColData(p ? p + n : p, pColData); + uint8_t *pBuf1 = NULL; + uint8_t *pBuf2 = NULL; + uint8_t *pBuf3 = NULL; + uint8_t *pBuf4 = NULL; + + // encode ================= + // columns AND SBlockCol + int32_t nBuf1 = 0; + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + + ASSERT(pColData->flag); + + if (pColData->flag == HAS_NONE) continue; + + SBlockCol blockCol = {.cid = pColData->cid, + .type = pColData->type, + .smaOn = pColData->smaOn, + .flag = pColData->flag, + .szOrigin = pColData->nData}; + + if (pColData->flag != HAS_NULL) { + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pBuf1, nBuf1, &pBuf3); + if (code) goto _exit; + + blockCol.offset = nBuf1; + nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + } + + code = tRealloc(&pBuf2, hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); + if (code) goto _exit; + hdr.szBlkCol += tPutBlockCol(pBuf2 + hdr.szBlkCol, &blockCol); } - return n; + int32_t nBuf2 = 0; + if (hdr.szBlkCol > 0) { + nBuf2 = hdr.szBlkCol + sizeof(TSCKSUM); + + code = tRealloc(&pBuf2, nBuf2); + if (code) goto _exit; + + taosCalcChecksumAppend(0, pBuf2, nBuf2); + } + + // uid + version + tskey + int32_t nBuf3 = 0; + if (pBlockData->uid == 0) { + code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, + &pBuf3, nBuf3, &hdr.szUid, &pBuf4); + if (code) goto _exit; + } + nBuf3 += hdr.szUid; + + code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, + cmprAlg, &pBuf3, nBuf3, &hdr.szVer, &pBuf4); + if (code) goto _exit; + nBuf3 += hdr.szVer; + + code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, + cmprAlg, &pBuf3, nBuf3, &hdr.szKey, &pBuf4); + if (code) goto _exit; + nBuf3 += hdr.szKey; + + nBuf3 += sizeof(TSCKSUM); + code = tRealloc(&pBuf3, nBuf3); + if (code) goto _exit; + + // hdr + int32_t nBuf4 = tPutDiskDataHdr(NULL, &hdr); + code = tRealloc(&pBuf4, nBuf4); + if (code) goto _exit; + tPutDiskDataHdr(pBuf4, &hdr); + taosCalcChecksumAppend(taosCalcChecksum(0, pBuf4, nBuf4), pBuf3, nBuf3); + + // aggragate + if (ppOut) { + *szOut = nBuf1 + nBuf2 + nBuf3 + nBuf4; + code = tRealloc(ppOut, *szOut); + if (code) goto _exit; + + memcpy(*ppOut, pBuf4, nBuf4); + memcpy(*ppOut + nBuf4, pBuf3, nBuf3); + if (nBuf2) { + memcpy(*ppOut + nBuf4 + nBuf3, pBuf2, nBuf2); + } + if (nBuf1) { + memcpy(*ppOut + nBuf4 + nBuf3 + nBuf2, pBuf1, nBuf1); + } + } + +_exit: + return code; } -int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData) { - int32_t n = 0; - - tBlockDataReset(pBlockData); - - n += tGetI32v(p + n, &pBlockData->nRow); - pBlockData->aVersion = (int64_t *)(p + n); - n = n + sizeof(int64_t) * pBlockData->nRow; - pBlockData->aTSKEY = (TSKEY *)(p + n); - n = n + sizeof(TSKEY) * pBlockData->nRow; - - int32_t nCol; - n += tGetI32v(p + n, &nCol); - for (int32_t iCol = 0; iCol < nCol; iCol++) { - SColData *pColData; - - if (tBlockDataAddColData(pBlockData, iCol, &pColData)) return -1; - n += tGetColData(p + n, pColData); - } - - return n; +int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData) { + int32_t code = 0; + // TODO + return code; } // SDiskDataHdr ============================== From 441ce21677594ff3562b4abb2a263549f1f16308 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 19 Aug 2022 17:19:50 +0800 Subject: [PATCH 143/194] enh: rsma batch process --- include/common/tmsg.h | 25 -- include/common/tmsgdef.h | 2 +- include/util/tqueue.h | 1 + source/dnode/mnode/impl/src/mndStb.c | 2 + source/dnode/vnode/src/inc/sma.h | 27 +- source/dnode/vnode/src/inc/vnodeInt.h | 1 - source/dnode/vnode/src/sma/smaCommit.c | 46 +-- source/dnode/vnode/src/sma/smaEnv.c | 13 +- source/dnode/vnode/src/sma/smaRollup.c | 462 +++++++++++------------- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 - source/libs/executor/src/executorimpl.c | 1 + source/util/src/tqueue.c | 3 +- 12 files changed, 253 insertions(+), 332 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f870bd161f..8f199c72f7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2667,31 +2667,6 @@ typedef struct { int32_t padding; } SRSmaExecMsg; -typedef struct { - int64_t suid; - int8_t level; -} SRSmaFetchMsg; - -static FORCE_INLINE int32_t tEncodeSRSmaFetchMsg(SEncoder* pCoder, const SRSmaFetchMsg* pReq) { - if (tStartEncode(pCoder) < 0) return -1; - - if (tEncodeI64(pCoder, pReq->suid) < 0) return -1; - if (tEncodeI8(pCoder, pReq->level) < 0) return -1; - - tEndEncode(pCoder); - return 0; -} - -static FORCE_INLINE int32_t tDecodeSRSmaFetchMsg(SDecoder* pCoder, SRSmaFetchMsg* pReq) { - if (tStartDecode(pCoder) < 0) return -1; - - if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1; - if (tDecodeI8(pCoder, &pReq->level) < 0) return -1; - - tEndDecode(pCoder); - return 0; -} - typedef struct { int8_t version; // for compatibility(default 0) int8_t intervalUnit; // MACRO: TIME_UNIT_XXX diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 16d5965759..e2bb3e2ae1 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -201,7 +201,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) - TD_DEF_MSG_TYPE(TDMT_VND_FETCH_RSMA, "vnode-fetch-rsma", SRSmaFetchMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_FETCH_RSMA, "vnode-fetch-rsma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_EXEC_RSMA, "vnode-exec-rsma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL) diff --git a/include/util/tqueue.h b/include/util/tqueue.h index 0f4f1db9ee..da409a90bb 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -76,6 +76,7 @@ void taosFreeQall(STaosQall *qall); int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall); int32_t taosGetQitem(STaosQall *qall, void **ppItem); void taosResetQitems(STaosQall *qall); +int32_t taosQallItemSize(STaosQall *qall); STaosQset *taosOpenQset(); void taosCloseQset(STaosQset *qset); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index e0f2b83160..ebec3d5ea6 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -442,6 +442,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt if (req.rollup) { req.rsmaParam.maxdelay[0] = pStb->maxdelay[0]; req.rsmaParam.maxdelay[1] = pStb->maxdelay[1]; + req.rsmaParam.watermark[0] = pStb->watermark[0]; + req.rsmaParam.watermark[1] = pStb->watermark[1]; if (pStb->ast1Len > 0) { if (mndConvertRsmaTask(&req.rsmaParam.qmsg[0], &req.rsmaParam.qmsgLen[0], pStb->pAst1, pStb->uid, STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[0]) < 0) { diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index c43772062e..989d24295e 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -90,14 +90,14 @@ struct SRSmaStat { SSma *pSma; int64_t commitAppliedVer; // vnode applied version for async commit int64_t refId; // shared by fetch tasks - volatile int64_t qBufSize; // queue buffer size + volatile int64_t nBufItems; // number of items in queue buffer SRWLatch lock; // r/w lock for rsma fs(e.g. qtaskinfo) + volatile int8_t nExecutor; // [1, max(half of query threads, 4)] int8_t triggerStat; // shared by fetch tasks int8_t commitStat; // 0 not in committing, 1 in committing - int8_t execStat; // 0 not in exec , 1 in exec SArray *aTaskFile; // qTaskFiles committed recently(for recovery/snapshot r/w) SHashObj *infoHash; // key: suid, value: SRSmaInfo - SHashObj *fetchHash; // key: suid, value: L1 or L2 or L1|L2 + tsem_t notEmpty; // has items in queue buffer }; struct SSmaStat { @@ -111,26 +111,29 @@ struct SSmaStat { #define SMA_STAT_TSMA(s) (&(s)->tsmaStat) #define SMA_STAT_RSMA(s) (&(s)->rsmaStat) #define RSMA_INFO_HASH(r) ((r)->infoHash) -#define RSMA_FETCH_HASH(r) ((r)->fetchHash) #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) #define RSMA_COMMIT_STAT(r) (&(r)->commitStat) #define RSMA_REF_ID(r) ((r)->refId) #define RSMA_FS_LOCK(r) (&(r)->lock) struct SRSmaInfoItem { - int8_t level; - int8_t triggerStat; - uint16_t interval; // second - int32_t maxDelay; - tmr_h tmrId; + int8_t level; + int8_t triggerStat; + uint8_t nSkipped; // number of skipped to fetch data from all active window + int8_t fetchLevel; + int32_t maxDelay; // ms + int64_t lastFetch; // ms + tmr_h tmrId; }; struct SRSmaInfo { STSchema *pTSchema; int64_t suid; - int64_t refId; // refId of SRSmaStat - uint64_t delFlag : 1; - uint64_t lastReceived : 63; // second + int64_t refId; // refId of SRSmaStat + int64_t lastRecv; // ms + int8_t delFlag; + int8_t assigned; // 0 idle, 1 assgined for exec + int16_t padding; T_REF_DECLARE() SRSmaInfoItem items[TSDB_RETENTION_L2]; void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7ac1cc4f0e..63d228ec8b 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -198,7 +198,6 @@ int32_t smaAsyncPreCommit(SSma* pSma); int32_t smaAsyncCommit(SSma* pSma); int32_t smaAsyncPostCommit(SSma* pSma); int32_t smaDoRetention(SSma* pSma, int64_t now); -int32_t smaProcessFetch(SSma* pSma, void* pMsg); int32_t smaProcessExec(SSma* pSma, void* pMsg); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 8b92475035..7b63056897 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -321,10 +321,10 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { int32_t nLoops = 0; while (1) { if (T_REF_VAL_GET(pStat) == 0) { - smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma commit, fetch tasks all finished", SMA_VID(pSma)); break; } else { - smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma commit, fetch tasks not all finished yet", SMA_VID(pSma)); } ++nLoops; if (nLoops > 1000) { @@ -338,30 +338,25 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { * 1) This is high cost task and should not put in asyncPreCommit originally. * 2) But, if put in asyncCommit, would trigger taskInfo cloning frequently. */ - nLoops = 0; - smaInfo("vgId:%d, start to wait for rsma qtask free, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(&pRSmaStat->execStat, 0, 1); - if (old == 0) break; - if (++nLoops > 1000) { - sched_yield(); - nLoops = 0; - smaDebug("vgId:%d, wait for rsma qtask free, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - } - } - - smaInfo("vgId:%d, end to wait for rsma qtask free, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - if (tdRSmaProcessExecImpl(pSma, RSMA_EXEC_COMMIT) < 0) { - atomic_store_8(&pRSmaStat->execStat, 0); return TSDB_CODE_FAILED; } + smaInfo("vgId:%d, rsma commit, wait for all items to be consumed, TID:%p", SMA_VID(pSma), (void*)taosGetSelfPthreadId()); + nLoops = 0; + while (atomic_load_64(&pRSmaStat->nBufItems) > 0) { + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void*)taosGetSelfPthreadId()); + +#if 0 // consuming task of qTaskInfo clone // step 4: swap queue/qall and iQueue/iQall // lock - taosWLockLatch(SMA_ENV_LOCK(pEnv)); + // taosWLockLatch(SMA_ENV_LOCK(pEnv)); ASSERT(RSMA_INFO_HASH(pRSmaStat)); @@ -376,11 +371,9 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter); } - atomic_store_64(&pRSmaStat->qBufSize, 0); - atomic_store_8(&pRSmaStat->execStat, 0); // unlock - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - + // taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); +#endif // step 5: others pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; @@ -426,7 +419,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { // step 1: merge qTaskInfo and iQTaskInfo // lock - taosWLockLatch(SMA_ENV_LOCK(pEnv)); + // taosWLockLatch(SMA_ENV_LOCK(pEnv)); void *pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); while (pIter) { @@ -480,10 +473,9 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { taosHashRemove(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t)); } taosArrayDestroy(rsmaDeleted); - // TODO: remove suid in files? // unlock - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + // taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); // step 2: cleanup outdated qtaskinfo files tdCleanupQTaskInfoFiles(pSma, pRSmaStat); diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index f51aad22bd..d39efc748e 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -209,6 +209,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat); pRSmaStat->pSma = (SSma *)pSma; atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT); + tsem_init(&pRSmaStat->notEmpty, 0, 0); // init smaMgmt smaInit(); @@ -230,12 +231,6 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS if (!RSMA_INFO_HASH(pRSmaStat)) { return TSDB_CODE_FAILED; } - - RSMA_FETCH_HASH(pRSmaStat) = taosHashInit( - RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); - if (!RSMA_FETCH_HASH(pRSmaStat)) { - return TSDB_CODE_FAILED; - } } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { // TODO } else { @@ -267,6 +262,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { smaDebug("vgId:%d, destroy rsma stat %p", SMA_VID(pSma), pRSmaStat); // step 1: set rsma trigger stat cancelled atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED); + tsem_destroy(&(pStat->notEmpty)); // step 2: destroy the rsma info and associated fetch tasks if (taosHashGetSize(RSMA_INFO_HASH(pStat)) > 0) { @@ -279,10 +275,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { } taosHashCleanup(RSMA_INFO_HASH(pStat)); - // step 3: destroy the rsma fetch hash - taosHashCleanup(RSMA_FETCH_HASH(pStat)); - - // step 4: wait all triggered fetch tasks finished + // step 3: wait all triggered fetch tasks finished int32_t nLoops = 0; while (1) { if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) { diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 9b3b0cb63d..e8018c0f33 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -15,10 +15,14 @@ #include "sma.h" -#define RSMA_QTASKINFO_BUFSIZE (32768) -#define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid -#define RSMA_QTASKEXEC_BUFSIZE (1048576) -#define RSMA_SUBMIT_BATCH_SIZE (1024) +#define RSMA_QTASKINFO_BUFSIZE (32768) // size +#define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid +#define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt +#define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt +#define RSMA_EXECUTOR_MAX (4) // cnt +#define RSMA_FETCH_DELAY_MAX (1800000) // ms +#define RSMA_FETCH_SKIP_MAX (1000) // cnt +#define RSMA_FETCH_ACTIVE_MAX (1800) // ms SSmaMgmt smaMgmt = { .inited = 0, @@ -40,11 +44,10 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSiz static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid); static void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); static void tdFreeRSmaSubmitItems(SArray *pItems); -static int32_t tdRSmaConsumeAndFetch(SSma *pSma, int64_t suid, int8_t level, SArray *pSubmitArr); +static int32_t tdRSmaConsumeAndFetch(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmitArr); static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid); static void tdRSmaFetchTrigger(void *param, void *tmrId); -static int32_t tdRSmaFetchSend(SSma *pSma, SRSmaInfo *pInfo, int8_t level); static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, int8_t type, SRSmaQTaskInfoIter *pIter); @@ -668,7 +671,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm } else { smaDebug("vgId:%d, rsma %" PRIi8 " data fetched", SMA_VID(pSma), pItem->level); } -#if 0 +#if 1 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, pItem->level); blockDebugShowDataBlocks(pResList, flag); @@ -729,22 +732,24 @@ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inpu taosWriteQitem(pInfo->queue, qItem); + pInfo->lastRecv = taosGetTimestampMs(); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); - int64_t bufSize = atomic_add_fetch_64(&pRSmaStat->qBufSize, pReq->header.contLen); + + tsem_post(&(pRSmaStat->notEmpty)); + + int64_t nItems = atomic_fetch_add_64(&pRSmaStat->nBufItems, 1); // smoothing consume - int32_t n = bufSize / RSMA_QTASKEXEC_BUFSIZE; + int32_t n = nItems / RSMA_QTASKEXEC_SMOOTH_SIZE; if (n > 1) { if (n > 10) { n = 10; } - taosMsleep(n << 4); - if (n > 2) { + taosMsleep(n << 3); + if (n > 5) { smaWarn("vgId:%d, pInfo->queue itemSize:%d, memSize:%" PRIi64 ", sleep %d ms", SMA_VID(pSma), - taosQueueItemSize(pInfo->queue), taosQueueMemorySize(pInfo->queue), n << 4); - } else { - smaDebug("vgId:%d, pInfo->queue itemSize:%d, memSize:%" PRIi64 ", sleep %d ms", SMA_VID(pSma), - taosQueueItemSize(pInfo->queue), taosQueueMemorySize(pInfo->queue), n << 4); + taosQueueItemSize(pInfo->queue), taosQueueMemorySize(pInfo->queue), n << 3); } } @@ -840,25 +845,25 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { return NULL; } - taosRLockLatch(SMA_ENV_LOCK(pEnv)); + // taosRLockLatch(SMA_ENV_LOCK(pEnv)); pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo && (pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + // taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } if (!pRSmaInfo->taskInfo[0]) { if (tdCloneRSmaInfo(pSma, pRSmaInfo) < 0) { - taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + // taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } } tdRefRSmaInfo(pSma, pRSmaInfo); - taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + // taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); ASSERT(pRSmaInfo->suid == suid); return pRSmaInfo; } - taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + // taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } @@ -910,22 +915,11 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp static int32_t tdRSmaExecCheck(SSma *pSma) { SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); - int64_t bufSize = atomic_load_64(&pRSmaStat->qBufSize); - if (bufSize < RSMA_QTASKEXEC_BUFSIZE) { - smaDebug("vgId:%d, bufSize is %d but has no chance to exec as less than %d", SMA_VID(pSma), bufSize, - RSMA_QTASKEXEC_BUFSIZE); + if (atomic_load_8(&pRSmaStat->nExecutor) >= TMIN(RSMA_EXECUTOR_MAX, tsNumOfVnodeQueryThreads / 2)) { return TSDB_CODE_SUCCESS; } - if (atomic_val_compare_exchange_8(&pRSmaStat->execStat, 0, 1) == 1) { - smaDebug("vgId:%d, bufSize is %d but has no chance to exec as qTaskInfo occupied by another task", SMA_VID(pSma), - bufSize); - return TSDB_CODE_SUCCESS; - } - - smaDebug("vgId:%d, bufSize is %d and has chance to exec as qTaskInfo is free now", SMA_VID(pSma), bufSize); - SRSmaExecMsg fetchMsg; int32_t contLen = sizeof(SMsgHead); void *pBuf = rpcMallocCont(0 + contLen); @@ -949,7 +943,6 @@ static int32_t tdRSmaExecCheck(SSma *pSma) { return TSDB_CODE_SUCCESS; _err: - atomic_store_8(&pRSmaStat->execStat, 0); return TSDB_CODE_FAILED; } @@ -959,7 +952,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { // only applicable when rsma env exists return TSDB_CODE_SUCCESS; } - + STbUidStore uidStore = {0}; SRetention *pRetention = SMA_RETENTION(pSma); if (!RETENTION_VALID(pRetention + 1)) { // return directly if retention level 1 is invalid @@ -967,25 +960,35 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { } if (inputType == STREAM_INPUT__DATA_SUBMIT) { - STbUidStore uidStore = {0}; - tdFetchSubmitReqSuids(pMsg, &uidStore); + if (tdFetchSubmitReqSuids(pMsg, &uidStore) < 0) { + goto _err; + } if (uidStore.suid != 0) { - tdExecuteRSmaAsync(pSma, pMsg, inputType, uidStore.suid); + if (tdExecuteRSmaAsync(pSma, pMsg, inputType, uidStore.suid) < 0) { + goto _err; + } void *pIter = taosHashIterate(uidStore.uidHash, NULL); while (pIter) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - tdExecuteRSmaAsync(pSma, pMsg, inputType, *pTbSuid); + if (tdExecuteRSmaAsync(pSma, pMsg, inputType, *pTbSuid) < 0) { + goto _err; + } pIter = taosHashIterate(uidStore.uidHash, pIter); } - tdUidStoreDestory(&uidStore); - - tdRSmaExecCheck(pSma); + if (tdRSmaExecCheck(pSma) < 0) { + goto _err; + } } } + tdUidStoreDestory(&uidStore); return TSDB_CODE_SUCCESS; +_err: + tdUidStoreDestory(&uidStore); + smaError("vgId:%d, failed to process rsma submit since: %s", SMA_VID(pSma), terrstr()); + return TSDB_CODE_FAILED; } /** @@ -1521,8 +1524,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); if (!pStat) { - smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%" PRIi64 " refId:%d)", - smaMgmt.rsetId, pRSmaInfo->refId); + smaWarn("rsma fetch task not start since rsma stat already destroyed, rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, + pRSmaInfo->refId); return; } @@ -1553,7 +1556,14 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { smaDebug("vgId:%d, rsma fetch task started for level:%" PRIi8 " suid:%" PRIi64 " since stat is active", SMA_VID(pSma), pItem->level, pRSmaInfo->suid); // async process - tdRSmaFetchSend(pSma, pRSmaInfo, pItem->level); + pItem->fetchLevel = pItem->level; + SRSmaInfo *qInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaInfo->suid); + SRSmaInfoItem *qItem = RSMA_INFO_ITEM(qInfo, pItem->level - 1); + ASSERT(qItem->level == pItem->level); + ASSERT(qItem->fetchLevel == pItem->fetchLevel); + tsem_post(&(pStat->notEmpty)); + smaInfo("vgId:%d, rsma fetch task planned for level:%" PRIi8 " suid:%" PRIi64, SMA_VID(pSma), pItem->level, + pRSmaInfo->suid); } break; case TASK_TRIGGER_STAT_PAUSED: { smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is paused", @@ -1568,8 +1578,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SMA_VID(pSma), pItem->level, pRSmaInfo->suid); } break; default: { - smaWarn("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is unknown", - SMA_VID(pSma), pItem->level, pRSmaInfo->suid); + smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is unknown", + SMA_VID(pSma), pItem->level, pRSmaInfo->suid); } break; } @@ -1578,135 +1588,21 @@ _end: tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); } -/** - * @brief put rsma fetch msg to fetch queue - * - * @param pSma - * @param pInfo - * @param level - * @return int32_t - */ -static int32_t tdRSmaFetchSend(SSma *pSma, SRSmaInfo *pInfo, int8_t level) { - SRSmaFetchMsg fetchMsg = {.suid = pInfo->suid, .level = level}; - int32_t ret = 0; - int32_t contLen = 0; - SEncoder encoder = {0}; - tEncodeSize(tEncodeSRSmaFetchMsg, &fetchMsg, contLen, ret); - if (ret < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tEncoderClear(&encoder); - goto _err; - } - - void *pBuf = rpcMallocCont(contLen + sizeof(SMsgHead)); - tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), contLen); - if (tEncodeSRSmaFetchMsg(&encoder, &fetchMsg) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tEncoderClear(&encoder); - } - tEncoderClear(&encoder); - - ((SMsgHead *)pBuf)->vgId = SMA_VID(pSma); - ((SMsgHead *)pBuf)->contLen = contLen + sizeof(SMsgHead); - - SRpcMsg rpcMsg = { - .code = 0, - .msgType = TDMT_VND_FETCH_RSMA, - .pCont = pBuf, - .contLen = contLen + sizeof(SMsgHead), - }; - - if ((terrno = tmsgPutToQueue(&pSma->pVnode->msgCb, QUERY_QUEUE, &rpcMsg)) != 0) { - smaError("vgId:%d, failed to put rsma fetch msg into fetch-queue for suid:%" PRIi64 " level:%" PRIi8 " since %s", - SMA_VID(pSma), pInfo->suid, level, terrstr()); - goto _err; - } - - smaDebug("vgId:%d, success to put rsma fetch msg into fetch-queue for suid:%" PRIi64 " level:%" PRIi8, SMA_VID(pSma), - pInfo->suid, level); - - return TSDB_CODE_SUCCESS; -_err: - return TSDB_CODE_FAILED; -} - -/** - * @brief fetch rsma data of level 2/3 and submit - * - * @param pSma - * @param pMsg - * @return int32_t - */ -int32_t smaProcessFetch(SSma *pSma, void *pMsg) { - SRpcMsg *pRpcMsg = (SRpcMsg *)pMsg; - SRSmaFetchMsg req = {0}; - SDecoder decoder = {0}; - void *pBuf = NULL; - SRSmaStat *pRSmaStat = NULL; - if (!pRpcMsg || pRpcMsg->contLen < sizeof(SMsgHead)) { - terrno = TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP; - goto _err; - } - - pBuf = POINTER_SHIFT(pRpcMsg->pCont, sizeof(SMsgHead)); - - tDecoderInit(&decoder, pBuf, pRpcMsg->contLen); - if (tDecodeSRSmaFetchMsg(&decoder, &req) < 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - - pRSmaStat = SMA_RSMA_STAT(pSma); - - if (atomic_val_compare_exchange_8(&pRSmaStat->execStat, 0, 1) == 0) { - SArray *pSubmitArr = NULL; - if (!(pSubmitArr = taosArrayInit(RSMA_SUBMIT_BATCH_SIZE, POINTER_BYTES))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - atomic_store_8(&pRSmaStat->execStat, 0); - goto _err; - } - tdRSmaConsumeAndFetch(pSma, req.suid, req.level, pSubmitArr); - atomic_store_8(&pRSmaStat->execStat, 0); - taosArrayDestroy(pSubmitArr); - } else { - int8_t level = req.level; - int8_t *val = taosHashGet(RSMA_FETCH_HASH(pRSmaStat), &req.suid, sizeof(req.suid)); - if (val) { - level |= (*val); - } - ASSERT(level >= 1 && level <= 3); - taosHashPut(RSMA_FETCH_HASH(pRSmaStat), &req.suid, sizeof(req.suid), &level, sizeof(level)); - } - - tDecoderClear(&decoder); - smaDebug("vgId:%d, success to process rsma fetch msg for suid:%" PRIi64 " level:%" PRIi8, SMA_VID(pSma), req.suid, - req.level); - return TSDB_CODE_SUCCESS; -_err: - tDecoderClear(&decoder); - smaError("vgId:%d, failed to process rsma fetch msg since %s", SMA_VID(pSma), terrstr()); - return TSDB_CODE_FAILED; -} - static void tdFreeRSmaSubmitItems(SArray *pItems) { for (int32_t i = 0; i < taosArrayGetSize(pItems); ++i) { taosFreeQitem(*(void **)taosArrayGet(pItems, i)); } } -static int32_t tdRSmaConsumeAndFetch(SSma *pSma, int64_t suid, int8_t level, SArray *pSubmitArr) { - SRSmaInfo *pInfo = tdAcquireRSmaInfoBySuid(pSma, suid); - if (!pInfo) { - return TSDB_CODE_SUCCESS; - } - +static int32_t tdRSmaConsumeAndFetch(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmitArr) { // step 1: consume submit req +#if 0 int64_t qMemSize = 0; if ((qMemSize = taosQueueMemorySize(pInfo->queue) > 0)) { taosReadAllQitems(pInfo->queue, pInfo->qall); // queue has mutex lock SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); - atomic_fetch_sub_64(&pRSmaStat->qBufSize, qMemSize); + atomic_fetch_sub_64(&pRSmaStat->nBufItems, taosQallItemSize(pInfo->qall)); taosArrayClear(pSubmitArr); @@ -1736,25 +1632,52 @@ static int32_t tdRSmaConsumeAndFetch(SSma *pSma, int64_t suid, int8_t level, SAr tdFreeRSmaSubmitItems(pSubmitArr); } } - - // step 2: fetch rsma result +#endif + // step 2: fetch rsma result(consider the efficiency and functionality) SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; for (int8_t i = 1; i <= TSDB_RETENTION_L2; ++i) { - if (level & i) { + SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, i - 1); + if (pItem->fetchLevel) { + pItem->fetchLevel = 0; qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pInfo, i - 1); if (!taskInfo) { continue; } + + int64_t curMs = taosGetTimestampMs(); + if ((pItem->nSkipped > RSMA_FETCH_SKIP_MAX) || (pItem->nSkipped * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) { + pItem->nSkipped = 0; + smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch executed", + SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay); + } else { + if (((curMs - pInfo->lastRecv) < RSMA_FETCH_ACTIVE_MAX)) { + ++pItem->nSkipped; + smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch skipped ", + SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); + continue; + } else { + smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ", + SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); + } + } + + pItem->lastFetch = curMs; + if ((terrno = qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) { goto _err; } - SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, i - 1); - if (tdRSmaFetchAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, suid) < 0) { + if (tdRSmaFetchAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid) < 0) { tdCleanupStreamInputDataBlock(taskInfo); goto _err; } tdCleanupStreamInputDataBlock(taskInfo); + smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch finished", + SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay); + } else { + smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 + " maxDelay:%d, fetch not executed as fetchLevel is %" PRIi8, + SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay, pItem->fetchLevel); } } @@ -1766,6 +1689,45 @@ _err: return TSDB_CODE_FAILED; } +static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SArray *pSubmitArr, ERsmaExecType type) { + taosArrayClear(pSubmitArr); + while (1) { + void *msg = NULL; + taosGetQitem(qall, (void **)&msg); + if (msg) { + if (taosArrayPush(pSubmitArr, &msg) < 0) { + tdFreeRSmaSubmitItems(pSubmitArr); + goto _err; + } + } else { + break; + } + } + + int32_t size = taosArrayGetSize(pSubmitArr); + if (size > 0) { + for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { + if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, type, i) < 0) { + tdFreeRSmaSubmitItems(pSubmitArr); + goto _err; + } + } + tdFreeRSmaSubmitItems(pSubmitArr); + } + return TSDB_CODE_SUCCESS; +_err: + while (1) { + void *msg = NULL; + taosGetQitem(qall, (void **)&msg); + if (msg) { + taosFreeQitem(msg); + } else { + break; + } + } + return TSDB_CODE_FAILED; +} + /** * @brief * @@ -1777,7 +1739,6 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SHashObj *infoHash = NULL; - SArray *pSubmitQArr = NULL; SArray *pSubmitArr = NULL; bool isFetchAll = false; @@ -1786,106 +1747,95 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { goto _err; } - if (type == RSMA_EXEC_OVERFLOW) { - taosRLockLatch(SMA_ENV_LOCK(pEnv)); - if (atomic_load_64(&pRSmaStat->qBufSize) < RSMA_QTASKEXEC_BUFSIZE) { - taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - return TSDB_CODE_SUCCESS; - } - taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - } - - if (!(pSubmitQArr = taosArrayInit(taosHashGetSize(infoHash), sizeof(SRSmaExecQItem)))) { + if (!(pSubmitArr = + taosArrayInit(TMIN(RSMA_SUBMIT_BATCH_SIZE, atomic_load_64(&pRSmaStat->nBufItems)), POINTER_BYTES))) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - if (!(pSubmitArr = taosArrayInit(RSMA_SUBMIT_BATCH_SIZE, POINTER_BYTES))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - // step 1: rsma exec - consume data in buffer queue for all suids - SRSmaExecQItem qItem = {0}; - void *pIter = taosHashIterate(infoHash, NULL); // infoHash has r/w lock - if (type == RSMA_EXEC_OVERFLOW) { - while (pIter) { - SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; - if (taosQueueItemSize(pInfo->queue)) { - taosReadAllQitems(pInfo->queue, pInfo->qall); // queue has mutex lock - qItem.qall = &pInfo->qall; - qItem.pRSmaInfo = pIter; - taosArrayPush(pSubmitQArr, &qItem); - } - ASSERT(taosQueueItemSize(pInfo->queue) == 0); - pIter = taosHashIterate(infoHash, pIter); - } - } else if (type == RSMA_EXEC_COMMIT) { - while (pIter) { - SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; - if (taosQueueItemSize(pInfo->iQueue)) { - taosReadAllQitems(pInfo->iQueue, pInfo->iQall); - qItem.qall = &pInfo->iQall; - qItem.pRSmaInfo = pIter; - taosArrayPush(pSubmitQArr, &qItem); - } - ASSERT(taosQueueItemSize(pInfo->iQueue) == 0); - pIter = taosHashIterate(infoHash, pIter); - } - } else { - ASSERT(0); - } - atomic_store_64(&pRSmaStat->qBufSize, 0); - - int32_t qSize = taosArrayGetSize(pSubmitQArr); - for (int32_t i = 0; i < qSize; ++i) { - SRSmaExecQItem *pItem = taosArrayGet(pSubmitQArr, i); - while (1) { - void *msg = NULL; - taosGetQitem(*(STaosQall **)pItem->qall, (void **)&msg); - if (msg) { - if (taosArrayPush(pSubmitArr, &msg) < 0) { - tdFreeRSmaSubmitItems(pSubmitArr); - goto _err; - } + int32_t nIdle = 0; + while (true) { + if (++nIdle > 100) { + if (atomic_fetch_sub_8(&pRSmaStat->nExecutor, 1) > 1) { + // free the exec thread if without SubmitReq + break; } else { + // keep at least 1 exec thread only if without SubmitReq in case of no query thread to use when busy again + atomic_add_fetch_8(&pRSmaStat->nExecutor, 1); + nIdle = 0; + } + } + // step 1: rsma exec - consume data in buffer queue for all suids + if (type == RSMA_EXEC_OVERFLOW || type == RSMA_EXEC_COMMIT) { + void *pIter = taosHashIterate(infoHash, NULL); // infoHash has r/w lock + while (pIter) { + SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; + int64_t itemSize = 0; + if ((itemSize = taosQueueItemSize(pInfo->queue)) || RSMA_INFO_ITEM(pInfo, 0)->fetchLevel || + RSMA_INFO_ITEM(pInfo, 1)->fetchLevel) { + smaDebug("vgId:%d queueItemSize is %" PRIi64 " execType:%" PRIi8, SMA_VID(pSma), itemSize, type); + if (atomic_val_compare_exchange_8(&pInfo->assigned, 0, 1) == 0) { + taosReadAllQitems(pInfo->queue, pInfo->qall); // queue has mutex lock + int32_t qallItemSize = taosQallItemSize(pInfo->qall); + if (qallItemSize > 0) { + tdRSmaBatchExec(pSma, pInfo, pInfo->qall, pSubmitArr, type); + } + + tdRSmaConsumeAndFetch(pSma, pInfo, pSubmitArr); + if (qallItemSize > 0) { + // subtract the item size after the task finished, commit should wait for all items be consumed + atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize); + nIdle = 0; + } + ASSERT(1 == atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0)); + } + } + pIter = taosHashIterate(infoHash, pIter); + } + if (type == RSMA_EXEC_COMMIT) { break; } } +#if 0 + else if (type == RSMA_EXEC_COMMIT) { + while (pIter) { + SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; + if (taosQueueItemSize(pInfo->iQueue)) { + if (atomic_val_compare_exchange_8(&pInfo->assigned, 0, 1) == 0) { + taosReadAllQitems(pInfo->iQueue, pInfo->iQall); // queue has mutex lock + int32_t qallItemSize = taosQallItemSize(pInfo->iQall); + if (qallItemSize > 0) { + atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize); + nIdle = 0; - int32_t size = taosArrayGetSize(pSubmitArr); - if (size > 0) { - SRSmaInfo *pInfo = *(SRSmaInfo **)pItem->pRSmaInfo; - for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { - if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, type, i) < 0) { - tdFreeRSmaSubmitItems(pSubmitArr); - goto _err; + // batch exec + tdRSmaBatchExec(pSma, pInfo, pInfo->qall, pSubmitArr, type); + } + + // tdRSmaConsumeAndFetch(pSma, pInfo, pSubmitArr); + ASSERT(1 == atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0)); + } } + ASSERT(taosQueueItemSize(pInfo->iQueue) == 0); + pIter = taosHashIterate(infoHash, pIter); } - tdFreeRSmaSubmitItems(pSubmitArr); - taosArrayClear(pSubmitArr); + break; + } +#endif + else { + ASSERT(0); } - } - // step 2: rsma fetch - consume data in buffer queue for suids triggered by timer - if (taosHashGetSize(RSMA_FETCH_HASH(pRSmaStat)) <= 0) { - goto _end; - } - pIter = taosHashIterate(RSMA_FETCH_HASH(pRSmaStat), NULL); - if (pIter) { - tdRSmaConsumeAndFetch(pSma, *(int64_t *)taosHashGetKey(pIter, NULL), *(int8_t *)pIter, pSubmitArr); - while ((pIter = taosHashIterate(RSMA_FETCH_HASH(pRSmaStat), pIter))) { - tdRSmaConsumeAndFetch(pSma, *(int64_t *)taosHashGetKey(pIter, NULL), *(int8_t *)pIter, pSubmitArr); + if (atomic_load_64(&pRSmaStat->nBufItems) <= 0) { + tsem_wait(&pRSmaStat->notEmpty); } - } + } // end of while(true) _end: taosArrayDestroy(pSubmitArr); - taosArrayDestroy(pSubmitQArr); return TSDB_CODE_SUCCESS; _err: taosArrayDestroy(pSubmitArr); - taosArrayDestroy(pSubmitQArr); return TSDB_CODE_FAILED; } @@ -1905,15 +1855,21 @@ int32_t smaProcessExec(SSma *pSma, void *pMsg) { goto _err; } smaDebug("vgId:%d, begin to process rsma exec msg by TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - if (tdRSmaProcessExecImpl(pSma, RSMA_EXEC_OVERFLOW) < 0) { - goto _err; + + int8_t nOld = atomic_fetch_add_8(&pRSmaStat->nExecutor, 1); + + if (nOld < TMIN(RSMA_EXECUTOR_MAX, tsNumOfVnodeQueryThreads / 2)) { + if (tdRSmaProcessExecImpl(pSma, RSMA_EXEC_OVERFLOW) < 0) { + goto _err; + } + } else { + atomic_fetch_sub_8(&pRSmaStat->nExecutor, 1); } - atomic_store_8(&pRSmaStat->execStat, 0); smaDebug("vgId:%d, success to process rsma exec msg by TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); return TSDB_CODE_SUCCESS; _err: - atomic_store_8(&pRSmaStat->execStat, 0); + atomic_fetch_sub_8(&pRSmaStat->nExecutor, 1); smaError("vgId:%d, failed to process rsma exec msg by TID:%p since %s", SMA_VID(pSma), (void *)taosGetSelfPthreadId(), terrstr()); return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index c73d2ccfd5..7a8d168f4f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -301,8 +301,6 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg, 0); case TDMT_SCH_QUERY_CONTINUE: return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg, 0); - case TDMT_VND_FETCH_RSMA: - return smaProcessFetch(pVnode->pSma, pMsg); case TDMT_VND_EXEC_RSMA: return smaProcessExec(pVnode->pSma, pMsg); default: diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d15dc99122..a432196b01 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3131,6 +3131,7 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) { initResultRow(resultRow); pInfo->resultRowInfo.cur = (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset}; + // releaseBufPage(pSup->pResultBuf, getBufPage(pSup->pResultBuf, pageId)); } if (offset != length) { diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 50beba8a9b..eb70002680 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -298,7 +298,8 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) { return num; } -void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } +void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } +int32_t taosQallItemSize(STaosQall *qall) { return qall->numOfItems; } STaosQset *taosOpenQset() { STaosQset *qset = taosMemoryCalloc(sizeof(STaosQset), 1); From 3b9d44e05a5f43eaa1ffb6bf8e272dc39a4a5fa4 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 19 Aug 2022 17:27:18 +0800 Subject: [PATCH 144/194] fix: check tags when it is not regular table --- source/libs/executor/src/executil.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index baeb972e05..bf969bf2e4 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -429,7 +429,9 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray for (int32_t i = 0; i < rows; i++) { int64_t* uid = taosArrayGet(uidList, i); void* tag = taosHashGet(tags, uid, sizeof(int64_t)); - ASSERT(tag); + if (suid != 0) { + ASSERT(tag); + } for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); From d3d03d8c4d9ae6914d19e339dab0cd6c0d466636 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 17:45:52 +0800 Subject: [PATCH 145/194] adjust more code --- source/dnode/vnode/src/inc/tsdb.h | 7 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 198 +++++++++--------- 2 files changed, 102 insertions(+), 103 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 9e02453ccf..f31a373254 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -574,7 +574,7 @@ struct SDelFWriter { SDelFile fDel; TdFilePtr pWriteH; - uint8_t *pBuf1; + uint8_t *aBuf[1]; }; struct SDataFWriter { @@ -591,10 +591,7 @@ struct SDataFWriter { SLastFile fLast; SSmaFile fSma; - uint8_t *pBuf1; - uint8_t *pBuf2; - uint8_t *pBuf3; - uint8_t *pBuf4; + uint8_t *aBuf[4]; }; struct STsdbReadSnap { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index dc9cf19593..f3911350e6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -75,7 +75,9 @@ int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { goto _err; } - tFree(pWriter->pBuf1); + for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree(pWriter->aBuf[iBuf]); + } taosMemoryFree(pWriter); *ppWriter = NULL; @@ -99,21 +101,21 @@ int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelId size += sizeof(TSCKSUM); // alloc - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // build n = 0; - n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - n += tPutDelData(pWriter->pBuf1 + n, taosArrayGet(aDelData, iDelData)); + n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData)); } - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pWriteH, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -147,21 +149,21 @@ int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { size += sizeof(TSCKSUM); // alloc - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // build n = 0; - n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - n += tPutDelIdx(pWriter->pBuf1 + n, taosArrayGet(aDelIdx, iDelIdx)); + n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx)); } - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pWriteH, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -215,7 +217,7 @@ struct SDelFReader { SDelFile fDel; TdFilePtr pReadH; - uint8_t *pBuf1; + uint8_t *aBuf[1]; }; int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { @@ -262,7 +264,9 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { code = TAOS_SYSTEM_ERROR(errno); goto _exit; } - tFree(pReader->pBuf1); + for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree(pReader->aBuf[iBuf]); + } taosMemoryFree(pReader); } *ppReader = NULL; @@ -286,11 +290,11 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData } // alloc - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // read - n = taosReadFile(pReader->pReadH, pReader->pBuf1, size); + n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -300,7 +304,7 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -309,10 +313,10 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData n = 0; uint32_t delimiter; - n += tGetU32(pReader->pBuf1 + n, &delimiter); + n += tGetU32(pReader->aBuf[0] + n, &delimiter); while (n < size - sizeof(TSCKSUM)) { SDelData delData; - n += tGetDelData(pReader->pBuf1 + n, &delData); + n += tGetDelData(pReader->aBuf[0] + n, &delData); if (taosArrayPush(aDelData, &delData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -344,11 +348,11 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { } // alloc - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // read - n = taosReadFile(pReader->pReadH, pReader->pBuf1, size); + n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -358,7 +362,7 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -366,13 +370,13 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { // decode n = 0; uint32_t delimiter; - n += tGetU32(pReader->pBuf1 + n, &delimiter); + n += tGetU32(pReader->aBuf[0] + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); while (n < size - sizeof(TSCKSUM)) { SDelIdx delIdx; - n += tGetDelIdx(pReader->pBuf1 + n, &delIdx); + n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx); if (taosArrayPush(aDelIdx, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -398,9 +402,7 @@ struct SDataFReader { TdFilePtr pLastFD; TdFilePtr pSmaFD; - uint8_t *pBuf1; - uint8_t *pBuf2; - uint8_t *pBuf3; + uint8_t *aBuf[3]; }; int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { @@ -483,9 +485,10 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { goto _err; } - tFree((*ppReader)->pBuf1); - tFree((*ppReader)->pBuf2); - tFree((*ppReader)->pBuf3); + for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree((*ppReader)->aBuf[iBuf]); + } + taosMemoryFree(*ppReader); _exit: @@ -510,7 +513,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { } // alloc - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // seek @@ -520,7 +523,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { } // read - n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); + n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -530,19 +533,19 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode n = 0; - n = tGetU32(pReader->pBuf1 + n, &delimiter); + n = tGetU32(pReader->aBuf[0] + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); while (n < size - sizeof(TSCKSUM)) { SBlockIdx blockIdx; - n += tGetBlockIdx(pReader->pBuf1 + n, &blockIdx); + n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx); if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -573,7 +576,7 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { } // alloc - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // seek @@ -583,7 +586,7 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { } // read - n = taosReadFile(pReader->pLastFD, pReader->pBuf1, size); + n = taosReadFile(pReader->pLastFD, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -593,19 +596,19 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } // decode n = 0; - n = tGetU32(pReader->pBuf1 + n, &delimiter); + n = tGetU32(pReader->aBuf[0] + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); while (n < size - sizeof(TSCKSUM)) { SBlockL blockl; - n += tGetBlockL(pReader->pBuf1 + n, &blockl); + n += tGetBlockL(pReader->aBuf[0] + n, &blockl); if (taosArrayPush(aBlockL, &blockl) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -631,7 +634,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl int64_t tn; // alloc - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // seek @@ -641,7 +644,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl } // read - n = taosReadFile(pReader->pHeadFD, pReader->pBuf1, size); + n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -651,7 +654,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -660,10 +663,10 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl n = 0; uint32_t delimiter; - n += tGetU32(pReader->pBuf1 + n, &delimiter); + n += tGetU32(pReader->aBuf[0] + n, &delimiter); ASSERT(delimiter == TSDB_FILE_DLMT); - tn = tGetMapData(pReader->pBuf1 + n, mBlock); + tn = tGetMapData(pReader->aBuf[0] + n, mBlock); if (tn < 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -688,11 +691,11 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD // alloc int32_t size = pSmaInfo->size + sizeof(TSCKSUM); - code = tRealloc(&pReader->pBuf1, size); + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; // read - int64_t n = taosReadFile(pReader->pSmaFD, pReader->pBuf1, size); + int64_t n = taosReadFile(pReader->pSmaFD, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -702,7 +705,7 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD } // check - if (!taosCheckChecksumWhole(pReader->pBuf1, size)) { + if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -712,7 +715,7 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD while (n < pSmaInfo->size) { SColumnDataAgg sma; - n += tGetColumnDataAgg(pReader->pBuf1 + n, &sma); + n += tGetColumnDataAgg(pReader->aBuf[0] + n, &sma); if (taosArrayPush(aColumnDataAgg, &sma) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -735,10 +738,10 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; // uid + version + tskey - code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->pBuf1, pBlkInfo->szKey, 1); + code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->aBuf[0], pBlkInfo->szKey, 1); if (code) goto _err; SDiskDataHdr hdr; - uint8_t *p = pReader->pBuf1 + tGetDiskDataHdr(pReader->pBuf1, &hdr); + uint8_t *p = pReader->aBuf[0] + tGetDiskDataHdr(pReader->aBuf[0], &hdr); ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(pBlockData->suid == hdr.suid); @@ -750,7 +753,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (hdr.uid == 0) { ASSERT(hdr.szUid); code = tsdbDecmprData(p, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aUid, - sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); + sizeof(int64_t) * hdr.nRow, &pReader->aBuf[1]); if (code) goto _err; } else { ASSERT(!hdr.szUid); @@ -759,24 +762,24 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo // version code = tsdbDecmprData(p, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aVersion, - sizeof(int64_t) * hdr.nRow, &pReader->pBuf2); + sizeof(int64_t) * hdr.nRow, &pReader->aBuf[1]); if (code) goto _err; p += hdr.szVer; // TSKEY code = tsdbDecmprData(p, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, - sizeof(TSKEY) * hdr.nRow, &pReader->pBuf2); + sizeof(TSKEY) * hdr.nRow, &pReader->aBuf[1]); if (code) goto _err; p += hdr.szKey; - ASSERT(p - pReader->pBuf1 == pBlkInfo->szKey - sizeof(TSCKSUM)); + ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey - sizeof(TSCKSUM)); // read and decode columns if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit; if (hdr.szBlkCol > 0) { int64_t offset = pBlkInfo->offset + pBlkInfo->szKey; - code = tsdbReadAndCheck(pFD, offset, &pReader->pBuf1, hdr.szBlkCol + sizeof(TSCKSUM), 1); + code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[0], hdr.szBlkCol + sizeof(TSCKSUM), 1); if (code) goto _err; } @@ -789,7 +792,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo while (pBlockCol && pBlockCol->cid < pColData->cid) { if (n < hdr.szBlkCol) { - n += tGetBlockCol(pReader->pBuf1 + n, pBlockCol); + n += tGetBlockCol(pReader->aBuf[0] + n, pBlockCol); } else { ASSERT(n == hdr.szBlkCol); pBlockCol = NULL; @@ -817,10 +820,10 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + pBlockCol->offset; int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); - code = tsdbReadAndCheck(pFD, offset, &pReader->pBuf2, size, 0); + code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[1], size, 0); if (code) goto _err; - code = tsdbDecmprColData(pReader->pBuf2, pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->pBuf3); + code = tsdbDecmprColData(pReader->aBuf[1], pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->aBuf[2]); if (code) goto _err; } } @@ -1096,10 +1099,9 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { goto _err; } - tFree((*ppWriter)->pBuf1); - tFree((*ppWriter)->pBuf2); - tFree((*ppWriter)->pBuf3); - tFree((*ppWriter)->pBuf4); + for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree((*ppWriter)->aBuf[iBuf]); + } taosMemoryFree(*ppWriter); _exit: *ppWriter = NULL; @@ -1210,21 +1212,21 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { size += sizeof(TSCKSUM); // alloc - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // build n = 0; - n = tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); + n = tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - n += tPutBlockIdx(pWriter->pBuf1 + n, taosArrayGet(aBlockIdx, iBlockIdx)); + n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx)); } - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1254,19 +1256,19 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBloc // alloc size = sizeof(uint32_t) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // build n = 0; - n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); - n += tPutMapData(pWriter->pBuf1 + n, mBlock); - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); + n += tPutMapData(pWriter->aBuf[0] + n, mBlock); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1308,21 +1310,21 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { size += sizeof(TSCKSUM); // alloc - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; // encode n = 0; - n += tPutU32(pWriter->pBuf1 + n, TSDB_FILE_DLMT); + n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { - n += tPutBlockL(pWriter->pBuf1 + n, taosArrayGet(aBlockL, iBlockL)); + n += tPutBlockL(pWriter->aBuf[0] + n, taosArrayGet(aBlockL, iBlockL)); } - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); ASSERT(n + sizeof(TSCKSUM) == size); // write - n = taosWriteFile(pWriter->pLastFD, pWriter->pBuf1, size); + n = taosWriteFile(pWriter->pLastFD, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1381,21 +1383,21 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SColumnDataAgg sma; tsdbCalcColDataSMA(pColData, &sma); - code = tRealloc(&pWriter->pBuf1, pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); + code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); if (code) goto _err; - pSmaInfo->size += tPutColumnDataAgg(pWriter->pBuf1 + pSmaInfo->size, &sma); + pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma); } // write if (pSmaInfo->size) { int32_t size = pSmaInfo->size + sizeof(TSCKSUM); - code = tRealloc(&pWriter->pBuf1, size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; - taosCalcChecksumAppend(0, pWriter->pBuf1, size); + taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->pBuf1, size); + int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1447,57 +1449,57 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock .szOrigin = pColData->nData}; if (pColData->flag != HAS_NULL) { - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pWriter->pBuf1, nBuf1, &pWriter->pBuf3); + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pWriter->aBuf[0], nBuf1, &pWriter->aBuf[2]); if (code) goto _err; blockCol.offset = nBuf1; nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } - code = tRealloc(&pWriter->pBuf2, hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); + code = tRealloc(&pWriter->aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); if (code) goto _err; - hdr.szBlkCol += tPutBlockCol(pWriter->pBuf2 + hdr.szBlkCol, &blockCol); + hdr.szBlkCol += tPutBlockCol(pWriter->aBuf[1] + hdr.szBlkCol, &blockCol); } int32_t nBuf2 = 0; if (hdr.szBlkCol > 0) { nBuf2 = hdr.szBlkCol + sizeof(TSCKSUM); - code = tRealloc(&pWriter->pBuf2, nBuf2); + code = tRealloc(&pWriter->aBuf[1], nBuf2); if (code) goto _err; - taosCalcChecksumAppend(0, pWriter->pBuf2, nBuf2); + taosCalcChecksumAppend(0, pWriter->aBuf[1], nBuf2); } // uid + version + tskey int32_t nBuf3 = 0; if (pBlockData->uid == 0) { code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pWriter->pBuf3, nBuf3, &hdr.szUid, &pWriter->pBuf4); + &pWriter->aBuf[2], nBuf3, &hdr.szUid, &pWriter->aBuf[3]); if (code) goto _err; } nBuf3 += hdr.szUid; code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &pWriter->pBuf3, nBuf3, &hdr.szVer, &pWriter->pBuf4); + cmprAlg, &pWriter->aBuf[2], nBuf3, &hdr.szVer, &pWriter->aBuf[3]); if (code) goto _err; nBuf3 += hdr.szVer; code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &pWriter->pBuf3, nBuf3, &hdr.szKey, &pWriter->pBuf4); + cmprAlg, &pWriter->aBuf[2], nBuf3, &hdr.szKey, &pWriter->aBuf[3]); if (code) goto _err; nBuf3 += hdr.szKey; nBuf3 += sizeof(TSCKSUM); - code = tRealloc(&pWriter->pBuf3, nBuf3); + code = tRealloc(&pWriter->aBuf[2], nBuf3); if (code) goto _err; // hdr int32_t nBuf4 = tPutDiskDataHdr(NULL, &hdr); - code = tRealloc(&pWriter->pBuf4, nBuf4); + code = tRealloc(&pWriter->aBuf[3], nBuf4); if (code) goto _err; - tPutDiskDataHdr(pWriter->pBuf4, &hdr); - taosCalcChecksumAppend(taosCalcChecksum(0, pWriter->pBuf4, nBuf4), pWriter->pBuf3, nBuf3); + tPutDiskDataHdr(pWriter->aBuf[3], &hdr); + taosCalcChecksumAppend(taosCalcChecksum(0, pWriter->aBuf[3], nBuf4), pWriter->aBuf[2], nBuf3); // write ================= TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; @@ -1505,20 +1507,20 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock pBlkInfo->szKey = nBuf4 + nBuf3; pBlkInfo->szBlock = nBuf1 + nBuf2 + nBuf3 + nBuf4; - int64_t n = taosWriteFile(pFD, pWriter->pBuf4, nBuf4); + int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], nBuf4); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = taosWriteFile(pFD, pWriter->pBuf3, nBuf3); + n = taosWriteFile(pFD, pWriter->aBuf[2], nBuf3); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } if (nBuf2) { - n = taosWriteFile(pFD, pWriter->pBuf2, nBuf2); + n = taosWriteFile(pFD, pWriter->aBuf[1], nBuf2); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1526,7 +1528,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock } if (nBuf1) { - n = taosWriteFile(pFD, pWriter->pBuf1, nBuf1); + n = taosWriteFile(pFD, pWriter->aBuf[0], nBuf1); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; From c21294c96f678e89ebb34762d185255b7ba8fc02 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Fri, 19 Aug 2022 17:48:20 +0800 Subject: [PATCH 146/194] feat(taosAdapter): taosAdapter supports Windows (#16237) * feat(taosAdapter): taosAdapter supports Windows * feat(taosAdapter): taosAdapter supports Windows --- tools/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index a79cbbe77d..5eec174618 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -174,15 +174,15 @@ ELSE () BUILD_COMMAND COMMAND set CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client COMMAND set CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib - COMMAND go build -ldflags "-s -w -X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" - COMMAND go build -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" + COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" + COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND COMMAND ${_upx_prefix}/src/upx/upx taosadapter.exe COMMAND cmake -E copy taosadapter.exe ${CMAKE_BINARY_DIR}/build/bin COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ - COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin + COMMAND cmake -E copy taosadapter-debug.exe ${CMAKE_BINARY_DIR}/build/bin ) unset(_upx_prefix) ELSE () From 47cddbfe2b4473a3a8d9ff3d81eb73714a1346b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 17:52:48 +0800 Subject: [PATCH 147/194] mroe code --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 67 ++++++++++++-------------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 740da45393..8355149292 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1500,7 +1500,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD *ppColData = NULL; } -int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut) { +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[]) { int32_t code = 0; SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, @@ -1510,14 +1510,9 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, .nRow = pBlockData->nRow, .cmprAlg = cmprAlg}; - uint8_t *pBuf1 = NULL; - uint8_t *pBuf2 = NULL; - uint8_t *pBuf3 = NULL; - uint8_t *pBuf4 = NULL; - // encode ================= // columns AND SBlockCol - int32_t nBuf1 = 0; + int32_t nBuf0 = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1532,71 +1527,71 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, .szOrigin = pColData->nData}; if (pColData->flag != HAS_NULL) { - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pBuf1, nBuf1, &pBuf3); + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &aBuf[0], nBuf0, &aBuf[2]); if (code) goto _exit; - blockCol.offset = nBuf1; - nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + blockCol.offset = nBuf0; + nBuf0 = nBuf0 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } - code = tRealloc(&pBuf2, hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); + code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); if (code) goto _exit; - hdr.szBlkCol += tPutBlockCol(pBuf2 + hdr.szBlkCol, &blockCol); + hdr.szBlkCol += tPutBlockCol(aBuf[1] + hdr.szBlkCol, &blockCol); } - int32_t nBuf2 = 0; + int32_t nBuf1 = 0; if (hdr.szBlkCol > 0) { - nBuf2 = hdr.szBlkCol + sizeof(TSCKSUM); + nBuf1 = hdr.szBlkCol + sizeof(TSCKSUM); - code = tRealloc(&pBuf2, nBuf2); + code = tRealloc(&aBuf[1], nBuf1); if (code) goto _exit; - taosCalcChecksumAppend(0, pBuf2, nBuf2); + taosCalcChecksumAppend(0, aBuf[1], nBuf1); } // uid + version + tskey - int32_t nBuf3 = 0; + int32_t nBuf2 = 0; if (pBlockData->uid == 0) { code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pBuf3, nBuf3, &hdr.szUid, &pBuf4); + &aBuf[2], nBuf2, &hdr.szUid, &aBuf[3]); if (code) goto _exit; } - nBuf3 += hdr.szUid; + nBuf2 += hdr.szUid; code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &pBuf3, nBuf3, &hdr.szVer, &pBuf4); + cmprAlg, &aBuf[2], nBuf2, &hdr.szVer, &aBuf[3]); if (code) goto _exit; - nBuf3 += hdr.szVer; + nBuf2 += hdr.szVer; code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &pBuf3, nBuf3, &hdr.szKey, &pBuf4); + cmprAlg, &aBuf[2], nBuf2, &hdr.szKey, &aBuf[3]); if (code) goto _exit; - nBuf3 += hdr.szKey; + nBuf2 += hdr.szKey; - nBuf3 += sizeof(TSCKSUM); - code = tRealloc(&pBuf3, nBuf3); + nBuf2 += sizeof(TSCKSUM); + code = tRealloc(&aBuf[2], nBuf2); if (code) goto _exit; // hdr - int32_t nBuf4 = tPutDiskDataHdr(NULL, &hdr); - code = tRealloc(&pBuf4, nBuf4); + int32_t nBuf3 = tPutDiskDataHdr(NULL, &hdr); + code = tRealloc(&aBuf[3], nBuf3); if (code) goto _exit; - tPutDiskDataHdr(pBuf4, &hdr); - taosCalcChecksumAppend(taosCalcChecksum(0, pBuf4, nBuf4), pBuf3, nBuf3); + tPutDiskDataHdr(aBuf[3], &hdr); + taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], nBuf3), aBuf[2], nBuf2); // aggragate if (ppOut) { - *szOut = nBuf1 + nBuf2 + nBuf3 + nBuf4; + *szOut = nBuf0 + nBuf1 + nBuf2 + nBuf3; code = tRealloc(ppOut, *szOut); if (code) goto _exit; - memcpy(*ppOut, pBuf4, nBuf4); - memcpy(*ppOut + nBuf4, pBuf3, nBuf3); - if (nBuf2) { - memcpy(*ppOut + nBuf4 + nBuf3, pBuf2, nBuf2); - } + memcpy(*ppOut, aBuf[3], nBuf3); + memcpy(*ppOut + nBuf3, aBuf[2], nBuf2); if (nBuf1) { - memcpy(*ppOut + nBuf4 + nBuf3 + nBuf2, pBuf1, nBuf1); + memcpy(*ppOut + nBuf3 + nBuf2, aBuf[1], nBuf1); + } + if (nBuf0) { + memcpy(*ppOut + nBuf3 + nBuf2 + nBuf1, aBuf[0], nBuf0); } } From 875566d36194ccb9d09a602810e85c42b3bee50e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 18:06:18 +0800 Subject: [PATCH 148/194] adjust more --- source/dnode/vnode/src/inc/tsdb.h | 2 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 93 ++----------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 57 ++++++------ 3 files changed, 41 insertions(+), 111 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f31a373254..9e58ac6711 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -152,6 +152,8 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **p int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], + int32_t aBufN[]); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index f3911350e6..1c5c9979b5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1424,111 +1424,38 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock pBlkInfo->szBlock = 0; pBlkInfo->szKey = 0; - // ================= DATA ==================== - SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, - .fmtVer = 0, - .suid = pBlockData->suid, - .uid = pBlockData->uid, - .nRow = pBlockData->nRow, - .cmprAlg = cmprAlg}; - - // encode ================= - // columns AND SBlockCol - int32_t nBuf1 = 0; - for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - ASSERT(pColData->flag); - - if (pColData->flag == HAS_NONE) continue; - - SBlockCol blockCol = {.cid = pColData->cid, - .type = pColData->type, - .smaOn = pColData->smaOn, - .flag = pColData->flag, - .szOrigin = pColData->nData}; - - if (pColData->flag != HAS_NULL) { - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &pWriter->aBuf[0], nBuf1, &pWriter->aBuf[2]); - if (code) goto _err; - - blockCol.offset = nBuf1; - nBuf1 = nBuf1 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); - } - - code = tRealloc(&pWriter->aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); - if (code) goto _err; - hdr.szBlkCol += tPutBlockCol(pWriter->aBuf[1] + hdr.szBlkCol, &blockCol); - } - - int32_t nBuf2 = 0; - if (hdr.szBlkCol > 0) { - nBuf2 = hdr.szBlkCol + sizeof(TSCKSUM); - - code = tRealloc(&pWriter->aBuf[1], nBuf2); - if (code) goto _err; - - taosCalcChecksumAppend(0, pWriter->aBuf[1], nBuf2); - } - - // uid + version + tskey - int32_t nBuf3 = 0; - if (pBlockData->uid == 0) { - code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &pWriter->aBuf[2], nBuf3, &hdr.szUid, &pWriter->aBuf[3]); - if (code) goto _err; - } - nBuf3 += hdr.szUid; - - code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &pWriter->aBuf[2], nBuf3, &hdr.szVer, &pWriter->aBuf[3]); + int32_t aBufN[4] = {0}; + code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN); if (code) goto _err; - nBuf3 += hdr.szVer; - - code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &pWriter->aBuf[2], nBuf3, &hdr.szKey, &pWriter->aBuf[3]); - if (code) goto _err; - nBuf3 += hdr.szKey; - - nBuf3 += sizeof(TSCKSUM); - code = tRealloc(&pWriter->aBuf[2], nBuf3); - if (code) goto _err; - - // hdr - int32_t nBuf4 = tPutDiskDataHdr(NULL, &hdr); - code = tRealloc(&pWriter->aBuf[3], nBuf4); - if (code) goto _err; - tPutDiskDataHdr(pWriter->aBuf[3], &hdr); - taosCalcChecksumAppend(taosCalcChecksum(0, pWriter->aBuf[3], nBuf4), pWriter->aBuf[2], nBuf3); // write ================= TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; - pBlkInfo->szKey = nBuf4 + nBuf3; - pBlkInfo->szBlock = nBuf1 + nBuf2 + nBuf3 + nBuf4; + pBlkInfo->szKey = aBufN[3] + aBufN[2]; + pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], nBuf4); + int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], aBufN[3]); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = taosWriteFile(pFD, pWriter->aBuf[2], nBuf3); + n = taosWriteFile(pFD, pWriter->aBuf[2], aBufN[2]); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (nBuf2) { - n = taosWriteFile(pFD, pWriter->aBuf[1], nBuf2); + if (aBufN[1]) { + n = taosWriteFile(pFD, pWriter->aBuf[1], aBufN[1]); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } } - if (nBuf1) { - n = taosWriteFile(pFD, pWriter->aBuf[0], nBuf1); + if (aBufN[0]) { + n = taosWriteFile(pFD, pWriter->aBuf[0], aBufN[0]); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8355149292..d1757a47b0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1500,7 +1500,8 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD *ppColData = NULL; } -int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[]) { +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], + int32_t aBufN[]) { int32_t code = 0; SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, @@ -1512,7 +1513,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, // encode ================= // columns AND SBlockCol - int32_t nBuf0 = 0; + aBufN[0] = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1527,11 +1528,11 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, .szOrigin = pColData->nData}; if (pColData->flag != HAS_NULL) { - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &aBuf[0], nBuf0, &aBuf[2]); + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &aBuf[0], aBufN[0], &aBuf[2]); if (code) goto _exit; - blockCol.offset = nBuf0; - nBuf0 = nBuf0 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + blockCol.offset = aBufN[0]; + aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); } code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); @@ -1539,59 +1540,59 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, hdr.szBlkCol += tPutBlockCol(aBuf[1] + hdr.szBlkCol, &blockCol); } - int32_t nBuf1 = 0; + aBufN[1] = 0; if (hdr.szBlkCol > 0) { - nBuf1 = hdr.szBlkCol + sizeof(TSCKSUM); + aBufN[1] = hdr.szBlkCol + sizeof(TSCKSUM); - code = tRealloc(&aBuf[1], nBuf1); + code = tRealloc(&aBuf[1], aBufN[1]); if (code) goto _exit; - taosCalcChecksumAppend(0, aBuf[1], nBuf1); + taosCalcChecksumAppend(0, aBuf[1], aBufN[1]); } // uid + version + tskey - int32_t nBuf2 = 0; + aBufN[2] = 0; if (pBlockData->uid == 0) { code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &aBuf[2], nBuf2, &hdr.szUid, &aBuf[3]); + &aBuf[2], aBufN[2], &hdr.szUid, &aBuf[3]); if (code) goto _exit; } - nBuf2 += hdr.szUid; + aBufN[2] += hdr.szUid; code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &aBuf[2], nBuf2, &hdr.szVer, &aBuf[3]); + cmprAlg, &aBuf[2], aBufN[2], &hdr.szVer, &aBuf[3]); if (code) goto _exit; - nBuf2 += hdr.szVer; + aBufN[2] += hdr.szVer; code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &aBuf[2], nBuf2, &hdr.szKey, &aBuf[3]); + cmprAlg, &aBuf[2], aBufN[2], &hdr.szKey, &aBuf[3]); if (code) goto _exit; - nBuf2 += hdr.szKey; + aBufN[2] += hdr.szKey; - nBuf2 += sizeof(TSCKSUM); - code = tRealloc(&aBuf[2], nBuf2); + aBufN[2] += sizeof(TSCKSUM); + code = tRealloc(&aBuf[2], aBufN[2]); if (code) goto _exit; // hdr - int32_t nBuf3 = tPutDiskDataHdr(NULL, &hdr); - code = tRealloc(&aBuf[3], nBuf3); + aBufN[3] = tPutDiskDataHdr(NULL, &hdr); + code = tRealloc(&aBuf[3], aBufN[3]); if (code) goto _exit; tPutDiskDataHdr(aBuf[3], &hdr); - taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], nBuf3), aBuf[2], nBuf2); + taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], aBufN[3]), aBuf[2], aBufN[2]); // aggragate if (ppOut) { - *szOut = nBuf0 + nBuf1 + nBuf2 + nBuf3; + *szOut = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; code = tRealloc(ppOut, *szOut); if (code) goto _exit; - memcpy(*ppOut, aBuf[3], nBuf3); - memcpy(*ppOut + nBuf3, aBuf[2], nBuf2); - if (nBuf1) { - memcpy(*ppOut + nBuf3 + nBuf2, aBuf[1], nBuf1); + memcpy(*ppOut, aBuf[3], aBufN[3]); + memcpy(*ppOut + aBufN[3], aBuf[2], aBufN[2]); + if (aBufN[1]) { + memcpy(*ppOut + aBufN[3] + aBufN[2], aBuf[1], aBufN[1]); } - if (nBuf0) { - memcpy(*ppOut + nBuf3 + nBuf2 + nBuf1, aBuf[0], nBuf0); + if (aBufN[0]) { + memcpy(*ppOut + aBufN[3] + aBufN[2] + aBufN[1], aBuf[0], aBufN[0]); } } From d251180faed7df87934a10a66a5513cbacf50d72 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 19 Aug 2022 17:24:45 +0800 Subject: [PATCH 149/194] refactor(stream): optimize interval --- source/libs/executor/inc/executorimpl.h | 3 - source/libs/executor/src/timewindowoperator.c | 70 ++----------------- 2 files changed, 7 insertions(+), 66 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 311d82c8a2..14f285abd7 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -924,9 +924,6 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); -SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 60822ce770..9453361a43 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -871,7 +871,6 @@ static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_ static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj* pUpdatedMap) { return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap); - ; } static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) { @@ -1595,7 +1594,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* downstream = pOperator->pDownstream[0]; SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -1886,62 +1885,6 @@ _error: return NULL; } -SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { - SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pOperator->pTaskInfo = pTaskInfo; - pInfo->inputOrder = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; - pInfo->execModel = OPTR_EXEC_MODEL_STREAM; - pInfo->win = pTaskInfo->window; - pInfo->twAggSup = *pTwAggSupp; - pInfo->primaryTsIndex = primaryTsSlotId; - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(&pOperator->resultInfo, numOfRows); - int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); - initBasicInfo(&pInfo->binfo, pResBlock); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); - - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo); - - pOperator->name = "StreamTimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL, - destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - destroyIntervalOperatorInfo(pInfo, numOfCols); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - // todo handle multiple timeline cases. assume no timeline interweaving static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -3109,11 +3052,12 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP); - SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); - TSKEY maxTs = INT64_MIN; + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + TSKEY maxTs = INT64_MIN; SExprSupp* pSup = &pOperator->exprSupp; From 13269cc5b5951625b936d636242e835c3b4a6d0a Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Fri, 19 Aug 2022 18:08:05 +0800 Subject: [PATCH 150/194] docs(docker): update docker document (#16253) * docs(docker): update docker document * docs(docker): update docker document --- docs/zh/14-reference/11-docker/index.md | 421 ++++++++++++------------ 1 file changed, 210 insertions(+), 211 deletions(-) diff --git a/docs/zh/14-reference/11-docker/index.md b/docs/zh/14-reference/11-docker/index.md index c03990ede2..743fc2d32f 100644 --- a/docs/zh/14-reference/11-docker/index.md +++ b/docs/zh/14-reference/11-docker/index.md @@ -25,10 +25,11 @@ curl -u root:taosdata -d "show databases" localhost:6041/rest/sql $ docker exec -it tdengine taos taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - log | 2022-01-17 13:57:22.270 | 10 | 1 | 1 | 1 | 10 | 30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | 0 | us | 0 | ready | -Query OK, 1 row(s) in set (0.002843s) + name | +================================= + information_schema | + performance_schema | +Query OK, 2 rows in database (0.033802s) ``` 因为运行在容器中的 TDengine 服务端使用容器的 hostname 建立连接，使用 taos shell 或者各种连接器（例如 JDBC-JNI）从容器外访问容器内的 TDengine 比较复杂，所以上述方式是访问容器中 TDengine 服务的最简单的方法，适用于一些简单场景。如果在一些复杂场景下想要从容器化使用 taos shell 或者各种连接器访问容器中的 TDengine 服务，请参考下一节。 @@ -45,10 +46,11 @@ docker run -d --name tdengine --network host tdengine/tdengine $ taos taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | myhost:6030 | 1 | 8 | ready | any | 2022-01-17 22:10:32.619 | | -Query OK, 1 row(s) in set (0.003233s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +================================================================================================================================================= + 1 | vm98:6030 | 0 | 32 | ready | 2022-08-19 14:50:05.337 | | +Query OK, 1 rows in database (0.010654s) + ``` ## 以指定的 hostname 和 port 启动 TDengine @@ -59,12 +61,13 @@ Query OK, 1 row(s) in set (0.003233s) docker run -d \ --name tdengine \ -e TAOS_FQDN=tdengine \ - -p 6030-6049:6030-6049 \ - -p 6030-6049:6030-6049/udp \ + -p 6030:6030 \ + -p 6041-6049:6041-6049 \ + -p 6041-6049:6041-6049/udp \ tdengine/tdengine ``` -上面的命令在容器中启动一个 TDengine 服务，其所监听的 hostname 为 tdengine ，并将容器的 6030 到 6049 端口段映射到主机的 6030 到 6049 端口段 （tcp 和 udp 都需要映射)。如果主机上该端口段已经被占用，可以修改上述命令指定一个主机上空闲的端口段。如果 `rpcForceTcp` 被设置为 `1` ，可以只映射 tcp 协议。 +上面的命令在容器中启动一个 TDengine 服务，其所监听的 hostname 为 tdengine ，并将容器的 6030 端口映射到主机的 6030 端口（TCP，只能映射主机 6030 端口），6041-6049 端口段映射到主机 6041-6049 端口段（tcp 和 udp 都需要映射，如果主机上该端口段已经被占用，可以修改上述命令指定一个主机上空闲的端口段）。 接下来，要确保 "tdengine" 这个 hostname 在 `/etc/hosts` 中可解析。 @@ -103,9 +106,9 @@ taos -h tdengine -P 6030 3. 在同一网络上的另一容器中启动 TDengine 客户端 ```shell - docker run --rm -it --network td-net -e TAOS_FIRST_EP=tdengine tdengine/tdengine taos + docker run --rm -it --network td-net -e TAOS_FIRST_EP=tdengine --entrypoint=taos tdengine/tdengine # or - #docker run --rm -it --network td-net -e tdengine/tdengine taos -h tdengine + #docker run --rm -it --network td-net --entrypoint=taos tdengine/tdengine -h tdengine ``` ## 在容器中启动客户端应用 @@ -115,7 +118,7 @@ taos -h tdengine -P 6030 ```docker FROM ubuntu:20.04 RUN apt-get update && apt-get install -y wget -ENV TDENGINE_VERSION=2.4.0.0 +ENV TDENGINE_VERSION=3.0.0.0 RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && cd TDengine-client-${TDENGINE_VERSION} \ @@ -129,6 +132,14 @@ RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_ 以下是一个 go 应用程序的示例： +* 创建 go mod 项目： + +```bash +go mod init app +``` + +* 创建 main.go： + ```go /* * In this test program, we'll create a database and insert 4 records then select out. @@ -212,12 +223,18 @@ func checkErr(err error, prompt string) { } ``` -如下是完整版本的 dockerfile +* 更新 go mod -```docker -FROM golang:1.17.6-buster as builder -ENV TDENGINE_VERSION=2.4.0.0 -RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ +```bash +go mod tidy +``` + +如下是完整版本的 dockerfile： + +```dockerfile +FROM golang:1.19.0-buster as builder +ENV TDENGINE_VERSION=3.0.0.0 +RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && cd TDengine-client-${TDENGINE_VERSION} \ && ./install_client.sh \ @@ -232,8 +249,8 @@ RUN go build FROM ubuntu:20.04 RUN apt-get update && apt-get install -y wget -ENV TDENGINE_VERSION=2.4.0.0 -RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ +ENV TDENGINE_VERSION=3.0.0.0 +RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && cd TDengine-client-${TDENGINE_VERSION} \ && ./install_client.sh \ @@ -248,113 +265,112 @@ CMD ["app"] 目前我们已经有了 `main.go`, `go.mod`, `go.sum`, `app.dockerfile`， 现在可以构建出这个应用程序并在 `td-net` 网络上启动它 ```shell -$ docker build -t app -f app.dockerfile -$ docker run --rm --network td-net app -h tdengine -p 6030 +$ docker build -t app -f app.dockerfile . +$ docker run --rm --network td-net app app -h tdengine -p 6030 ============= args parse result: ============= hostName: tdengine serverPort: 6030 usr: root password: taosdata ================================================ -2022-01-17 15:56:55.48 +0000 UTC 0 -2022-01-17 15:56:56.48 +0000 UTC 1 -2022-01-17 15:56:57.48 +0000 UTC 2 -2022-01-17 15:56:58.48 +0000 UTC 3 -2022-01-17 15:58:01.842 +0000 UTC 0 -2022-01-17 15:58:02.842 +0000 UTC 1 -2022-01-17 15:58:03.842 +0000 UTC 2 -2022-01-17 15:58:04.842 +0000 UTC 3 -2022-01-18 01:43:48.029 +0000 UTC 0 -2022-01-18 01:43:49.029 +0000 UTC 1 -2022-01-18 01:43:50.029 +0000 UTC 2 -2022-01-18 01:43:51.029 +0000 UTC 3 +2022-08-19 07:43:51.68 +0000 UTC 0 +2022-08-19 07:43:52.68 +0000 UTC 1 +2022-08-19 07:43:53.68 +0000 UTC 2 +2022-08-19 07:43:54.68 +0000 UTC 3 ``` ## 用 docker-compose 启动 TDengine 集群 -1. 如下 docker-compose 文件启动一个 2 副本、2 管理节点、2 数据节点以及 1 个 arbitrator 的 TDengine 集群。 +1. 如下 docker-compose 文件启动一个 三节点 TDengine 集群。 - ```docker - version: "3" - services: - arbitrator: - image: tdengine/tdengine:$VERSION - command: tarbitrator - td-1: - image: tdengine/tdengine:$VERSION - environment: - TAOS_FQDN: "td-1" - TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 - volumes: - - taosdata-td1:/var/lib/taos/ - - taoslog-td1:/var/log/taos/ - td-2: - image: tdengine/tdengine:$VERSION - environment: - TAOS_FQDN: "td-2" - TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 - volumes: - - taosdata-td2:/var/lib/taos/ - - taoslog-td2:/var/log/taos/ - volumes: - taosdata-td1: - taoslog-td1: - taosdata-td2: - taoslog-td2: - ``` +```yml +version: "3" +services: + td-1: + image: tdengine/tdengine:$VERSION + environment: + TAOS_FQDN: "td-1" + TAOS_FIRST_EP: "td-1" + volumes: + - taosdata-td1:/var/lib/taos/ + - taoslog-td1:/var/log/taos/ + td-2: + image: tdengine/tdengine:$VERSION + environment: + TAOS_FQDN: "td-2" + TAOS_FIRST_EP: "td-1" + volumes: + - taosdata-td2:/var/lib/taos/ + - taoslog-td2:/var/log/taos/ + td-3: + image: tdengine/tdengine:$VERSION + environment: + TAOS_FQDN: "td-3" + TAOS_FIRST_EP: "td-1" + volumes: + - taosdata-td3:/var/lib/taos/ + - taoslog-td3:/var/log/taos/ +volumes: + taosdata-td1: + taoslog-td1: + taosdata-td2: + taoslog-td2: + taosdata-td3: + taoslog-td3: +``` :::note -- `VERSION` 环境变量被用来设置 tdengine image tag -- 在新创建的实例上必须设置 `TAOS_FIRST_EP` 以使其能够加入 TDengine 集群；如果有高可用需求，则需要同时使用 `TAOS_SECOND_EP` -- `TAOS_REPLICA` 用来设置缺省的数据库副本数量，其取值范围为[1,3] - 在双副本环境下，推荐使用 arbitrator, 用 TAOS_ARBITRATOR 来设置 - ::: +* `VERSION` 环境变量被用来设置 tdengine image tag +* 在新创建的实例上必须设置 `TAOS_FIRST_EP` 以使其能够加入 TDengine 集群；如果有高可用需求，则需要同时使用 `TAOS_SECOND_EP` +::: 2. 启动集群 - ```shell - $ VERSION=2.4.0.0 docker-compose up -d - Creating network "test_default" with the default driver - Creating volume "test_taosdata-td1" with default driver - Creating volume "test_taoslog-td1" with default driver - Creating volume "test_taosdata-td2" with default driver - Creating volume "test_taoslog-td2" with default driver - Creating test_td-1_1 ... done - Creating test_arbitrator_1 ... done - Creating test_td-2_1 ... done - ``` +```shell +$ VERSION=3.0.0.0 docker-compose up -d +Creating network "test-docker_default" with the default driver +Creating volume "test-docker_taosdata-td1" with default driver +Creating volume "test-docker_taoslog-td1" with default driver +Creating volume "test-docker_taosdata-td2" with default driver +Creating volume "test-docker_taoslog-td2" with default driver +Creating volume "test-docker_taosdata-td3" with default driver +Creating volume "test-docker_taoslog-td3" with default driver + +Creating test-docker_td-3_1 ... done +Creating test-docker_td-1_1 ... done +Creating test-docker_td-2_1 ... done +``` 3. 查看节点状态 - ```shell - $ docker-compose ps - Name Command State Ports - --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - test_arbitrator_1 /usr/bin/entrypoint.sh tar ... Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp - test_td-1_1 /usr/bin/entrypoint.sh taosd Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp - test_td-2_1 /usr/bin/entrypoint.sh taosd Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp - ``` +```shell + docker-compose ps + Name Command State Ports + +------------------------------------------------------------------- +test-docker_td-1_1 /tini -- /usr/bin/entrypoi ... Up +test-docker_td-2_1 /tini -- /usr/bin/entrypoi ... Up +test-docker_td-3_1 /tini -- /usr/bin/entrypoi ... Up +``` 4. 用 taos shell 查看 dnodes - ```shell - $ docker-compose exec td-1 taos -s "show dnodes" +```shell - taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | - ====================================================================================================================================== - 1 | td-1:6030 | 1 | 8 | ready | any | 2022-01-18 02:47:42.871 | | - 2 | td-2:6030 | 0 | 8 | ready | any | 2022-01-18 02:47:43.518 | | - 0 | arbitrator:6042 | 0 | 0 | ready | arb | 2022-01-18 02:47:43.633 | - | - Query OK, 3 row(s) in set (0.000811s) - ``` +$ docker-compose exec td-1 taos -s "show dnodes" + +taos> show dnodes + + id | endpoint | vnodes | support_vnodes | status | create_time | note | +================================================================================================================================================= + + 1 | td-1:6030 | 0 | 32 | ready | 2022-08-19 07:57:29.971 | | + 2 | td-2:6030 | 0 | 32 | ready | 2022-08-19 07:57:31.415 | | + 3 | td-3:6030 | 0 | 32 | ready | 2022-08-19 07:57:31.417 | | +Query OK, 3 rows in database (0.021262s) + +``` ## taosAdapter @@ -362,93 +378,80 @@ password: taosdata 2. 同时为了部署灵活起见，可以在独立的容器中启动 taosAdapter - ```docker - services: - # ... - adapter: - image: tdengine/tdengine:$VERSION - command: taosadapter - ``` +```docker +services: + # ... + adapter: + image: tdengine/tdengine:$VERSION + command: taosadapter +``` - 如果要部署多个 taosAdapter 来提高吞吐量并提供高可用性，推荐配置方式为使用 nginx 等反向代理来提供统一的访问入口。具体配置方法请参考 nginx 的官方文档。如下是示例： +如果要部署多个 taosAdapter 来提高吞吐量并提供高可用性，推荐配置方式为使用 nginx 等反向代理来提供统一的访问入口。具体配置方法请参考 nginx 的官方文档。如下是示例： - ```docker - version: "3" +```yml +version: "3" - networks: - inter: - api: +networks: + inter: - services: - arbitrator: - image: tdengine/tdengine:$VERSION - command: tarbitrator - networks: - - inter - td-1: - image: tdengine/tdengine:$VERSION - networks: - - inter - environment: - TAOS_FQDN: "td-1" - TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 - volumes: - - taosdata-td1:/var/lib/taos/ - - taoslog-td1:/var/log/taos/ - td-2: - image: tdengine/tdengine:$VERSION - networks: - - inter - environment: - TAOS_FQDN: "td-2" - TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 - volumes: - - taosdata-td2:/var/lib/taos/ - - taoslog-td2:/var/log/taos/ - adapter: - image: tdengine/tdengine:$VERSION - command: taosadapter - networks: - - inter - environment: - TAOS_FIRST_EP: "td-1" - TAOS_SECOND_EP: "td-2" - deploy: - replicas: 4 - nginx: - image: nginx - depends_on: - - adapter - networks: - - inter - - api - ports: - - 6041:6041 - - 6044:6044/udp - command: [ - "sh", - "-c", - "while true; - do curl -s http://adapter:6041/-/ping >/dev/null && break; - done; - printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}' - > /etc/nginx/conf.d/rest.conf; - printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}' - >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf; - nginx -g 'daemon off;'", - ] - volumes: - taosdata-td1: - taoslog-td1: - taosdata-td2: - taoslog-td2: - ``` +services: + td-1: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-1" + TAOS_FIRST_EP: "td-1" + volumes: + - taosdata-td1:/var/lib/taos/ + - taoslog-td1:/var/log/taos/ + td-2: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-2" + TAOS_FIRST_EP: "td-1" + volumes: + - taosdata-td2:/var/lib/taos/ + - taoslog-td2:/var/log/taos/ + adapter: + image: tdengine/tdengine:$VERSION + entrypoint: "taosadapter" + networks: + - inter + environment: + TAOS_FIRST_EP: "td-1" + TAOS_SECOND_EP: "td-2" + deploy: + replicas: 4 + nginx: + image: nginx + depends_on: + - adapter + networks: + - inter + ports: + - 6041:6041 + - 6044:6044/udp + command: [ + "sh", + "-c", + "while true; + do curl -s http://adapter:6041/-/ping >/dev/null && break; + done; + printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}' + > /etc/nginx/conf.d/rest.conf; + printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}' + >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf; + nginx -g 'daemon off;'", + ] +volumes: + taosdata-td1: + taoslog-td1: + taosdata-td2: + taoslog-td2: +``` ## 使用 docker swarm 部署 @@ -457,50 +460,46 @@ password: taosdata docker-compose 文件可以参考上节。下面是使用 docker swarm 启动 TDengine 的命令： ```shell -$ VERSION=2.4.0 docker stack deploy -c docker-compose.yml taos +$ VERSION=3.0.0.0 docker stack deploy -c docker-compose.yml taos Creating network taos_inter -Creating network taos_api -Creating service taos_arbitrator +Creating service taos_nginx Creating service taos_td-1 Creating service taos_td-2 Creating service taos_adapter -Creating service taos_nginx ``` 查看和管理 ```shell $ docker stack ps taos -ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS -79ni8temw59n taos_nginx.1 nginx:latest TM1701 Running Running about a minute ago -3e94u72msiyg taos_adapter.1 tdengine/tdengine:2.4.0 TM1702 Running Running 56 seconds ago -100amjkwzsc6 taos_td-2.1 tdengine/tdengine:2.4.0 TM1703 Running Running about a minute ago -pkjehr2vvaaa taos_td-1.1 tdengine/tdengine:2.4.0 TM1704 Running Running 2 minutes ago -tpzvgpsr1qkt taos_arbitrator.1 tdengine/tdengine:2.4.0 TM1705 Running Running 2 minutes ago -rvss3g5yg6fa taos_adapter.2 tdengine/tdengine:2.4.0 TM1706 Running Running 56 seconds ago -i2augxamfllf taos_adapter.3 tdengine/tdengine:2.4.0 TM1707 Running Running 56 seconds ago -lmjyhzccpvpg taos_adapter.4 tdengine/tdengine:2.4.0 TM1708 Running Running 56 seconds ago +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +7m3sbf532bqp taos_adapter.1 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago +pj403n6ofmmh taos_adapter.2 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago +rxqfwsyk5q1h taos_adapter.3 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago +qj40lpxr40oc taos_adapter.4 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago +oe3455ulxpze taos_nginx.1 nginx:latest vm98 Running Running about a minute ago +o0tsg70nrrc6 taos_td-1.1 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago +q5m1oxs589cp taos_td-2.1 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago $ docker service ls -ID NAME MODE REPLICAS IMAGE PORTS -561t4lu6nfw6 taos_adapter replicated 4/4 tdengine/tdengine:2.4.0 -3hk5ct3q90sm taos_arbitrator replicated 1/1 tdengine/tdengine:2.4.0 -d8qr52envqzu taos_nginx replicated 1/1 nginx:latest *:6041->6041/tcp, *:6044->6044/udp -2isssfvjk747 taos_td-1 replicated 1/1 tdengine/tdengine:2.4.0 -9pzw7u02ichv taos_td-2 replicated 1/1 tdengine/tdengine:2.4.0 +ID NAME MODE REPLICAS IMAGE PORTS +ozuklorgl8bs taos_adapter replicated 4/4 tdengine/tdengine:3.0.0.0 +crmhdjw6vxw0 taos_nginx replicated 1/1 nginx:latest *:6041->6041/tcp, *:6044->6044/udp +o86ngy7csv5n taos_td-1 replicated 1/1 tdengine/tdengine:3.0.0.0 +rma040ny4tb0 taos_td-2 replicated 1/1 tdengine/tdengine:3.0.0.0 ``` -从上面的输出可以看到有两个 dnode， 和两个 taosAdapter，以及一个 nginx 反向代理服务。 +从上面的输出可以看到有两个 dnode， 和四个 taosAdapter，以及一个 nginx 反向代理服务。 接下来，我们可以减少 taosAdapter 服务的数量 ```shell $ docker service scale taos_adapter=1 taos_adapter scaled to 1 -overall progress: 1 out of 1 tasks -1/1: running [==================================================>] +overall progress: 1 out of 1 tasks +1/1: running [==================================================>] verify: Service converged $ docker service ls -f name=taos_adapter -ID NAME MODE REPLICAS IMAGE PORTS -561t4lu6nfw6 taos_adapter replicated 1/1 tdengine/tdengine:2.4.0 +ID NAME MODE REPLICAS IMAGE PORTS +ozuklorgl8bs taos_adapter replicated 1/1 tdengine/tdengine:3.0.0.0 ``` From 0993843c8115c89aad0e431e08c69631c541e00b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 19 Aug 2022 18:13:01 +0800 Subject: [PATCH 151/194] fix(query): fix "n","y" time unit parsing does not return error --- source/libs/parser/src/parTranslater.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1c7446ad6f..9486671b16 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2822,6 +2822,29 @@ static int32_t createDefaultFillNode(STranslateContext* pCxt, SNode** pOutput) { return TSDB_CODE_SUCCESS; } +static int32_t checkEvery(STranslateContext* pCxt, SValueNode* pInterval) { + int32_t len = strlen(pInterval->literal); + + char *unit = &pInterval->literal[len - 1]; + if (*unit == 'n' || *unit == 'y') { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, + "Unsupported time unit in Every clause"); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateInterpEvery(STranslateContext* pCxt, SNode** pEvery) { + int32_t code = TSDB_CODE_SUCCESS; + + code = checkEvery(pCxt, (SValueNode *)(*pEvery)); + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, pEvery); + } + + return code; +} + static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect) { int32_t code = TSDB_CODE_SUCCESS; @@ -2856,7 +2879,7 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { int32_t code = translateExpr(pCxt, &pSelect->pRange); if (TSDB_CODE_SUCCESS == code) { - code = translateExpr(pCxt, &pSelect->pEvery); + code = translateInterpEvery(pCxt, &pSelect->pEvery); } if (TSDB_CODE_SUCCESS == code) { code = translateInterpFill(pCxt, pSelect); From 02055c4a90685021abb960bb750ee454b5d782eb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 19 Aug 2022 18:16:46 +0800 Subject: [PATCH 152/194] fix error msg --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9486671b16..3c0d9a5f63 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2828,7 +2828,7 @@ static int32_t checkEvery(STranslateContext* pCxt, SValueNode* pInterval) { char *unit = &pInterval->literal[len - 1]; if (*unit == 'n' || *unit == 'y') { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, - "Unsupported time unit in Every clause"); + "Unsupported time unit in EVERY clause"); } return TSDB_CODE_SUCCESS; From 4acc76b7c54af6a1fa8f26ab8ce28ebe9f0834fa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 18:35:09 +0800 Subject: [PATCH 153/194] adjust more code --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbUtil.c | 71 +++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 9e58ac6711..d1f5cfb122 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -154,6 +154,7 @@ int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlo int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], int32_t aBufN[]); +int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d1757a47b0..6db9d5e6f4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1600,9 +1600,76 @@ _exit: return code; } -int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData) { +int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]) { int32_t code = 0; - // TODO + + tBlockDataClear(pBlockData); + + int32_t n = 0; + SDiskDataHdr hdr = {0}; + + // SDiskDataHdr + n += tGetDiskDataHdr(pIn + n, &hdr); + if (!taosCheckChecksumWhole(pIn, n + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM))) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + ASSERT(hdr.delimiter == TSDB_FILE_DLMT); + + pBlockData->suid = hdr.suid; + pBlockData->uid = hdr.uid; + pBlockData->nRow = hdr.nRow; + + // uid + if (hdr.uid == 0) { + ASSERT(hdr.szUid); + code = tsdbDecmprData(pIn + n, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aUid, + sizeof(int64_t) * hdr.nRow, &aBuf[0]); + if (code) goto _exit; + } else { + ASSERT(!hdr.szUid); + } + n += hdr.szUid; + + // version + code = tsdbDecmprData(pIn + n, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aVersion, + sizeof(int64_t) * hdr.nRow, &aBuf[0]); + if (code) goto _exit; + n += hdr.szVer; + + // TSKEY + code = tsdbDecmprData(pIn + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, + sizeof(TSKEY) * hdr.nRow, &aBuf[0]); + if (code) goto _exit; + n = n + hdr.szKey + sizeof(TSCKSUM); + + // loop to decode each column data + if (hdr.szBlkCol == 0) goto _exit; + + int32_t nt = 0; + while (nt < hdr.szBlkCol) { + SBlockCol blockCol = {0}; + nt += tGetBlockCol(pIn + n + nt, &blockCol); + ASSERT(nt <= hdr.szBlkCol); + + SColData *pColData; + code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); + if (code) goto _exit; + + tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); + if (blockCol.flag == HAS_NULL) { + for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type)); + if (code) goto _exit; + } + } else { + code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &blockCol, hdr.cmprAlg, + hdr.nRow, pColData, &aBuf[0]); + if (code) goto _exit; + } + } + +_exit: return code; } From 03385305443bbf1ae3c32b76bdfbedfd7ec05ae9 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 19 Aug 2022 19:40:21 +0800 Subject: [PATCH 154/194] fix: wait pPageH to complete initialization --- source/libs/tdb/src/db/tdbPCache.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 22229ea0e8..ab9b21dc3f 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -199,10 +199,20 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) if (pPageH) { // copy the page content memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid)); + + for (int nLoops = 0;;) { + if (pPageH->pPager) break; + if (++nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + pPage->pLruNext = NULL; pPage->pPager = pPageH->pPager; memcpy(pPage->pData, pPageH->pData, pPage->pageSize); + tdbDebug("pcache/pPageH: %p %d %p %p", pPageH, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize, pPage); tdbPageInit(pPage, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize); pPage->kLen = pPageH->kLen; pPage->vLen = pPageH->vLen; From a9fcc12c338e9af562213e2a8fc54909a99a7e7d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 19 Aug 2022 20:08:09 +0800 Subject: [PATCH 155/194] enh: rsma batch process --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 6 +++- source/dnode/vnode/src/inc/sma.h | 3 +- source/dnode/vnode/src/inc/vnodeInt.h | 3 ++ source/dnode/vnode/src/sma/smaOpen.c | 11 ++++++ source/dnode/vnode/src/sma/smaRollup.c | 36 +++++++++++-------- source/dnode/vnode/src/vnd/vnodeOpen.c | 3 ++ .../tsim/sma/rsmaPersistenceRecovery.sim | 6 ++++ 7 files changed, 52 insertions(+), 16 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 1f981cc9e0..4987aa42e1 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -87,10 +87,14 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { while (pVnode->refCount > 0) taosMsleep(10); dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId); + while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); + while (!taosQueueEmpty(pVnode->pQueryQ)) { + taosMsleep(10); + dInfo("prop:vgId:%d, query queue size is %d", pVnode->vgId, taosQueueItemSize(pVnode->pQueryQ)); + } while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10); dTrace("vgId:%d, vnode queue is empty", pVnode->vgId); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 989d24295e..d32d67e29c 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -32,7 +32,8 @@ extern "C" { #define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on -#define RSMA_TASK_INFO_HASH_SLOT 8 +#define RSMA_TASK_INFO_HASH_SLOT (8) +#define RSMA_EXECUTOR_MAX (4) typedef struct SSmaEnv SSmaEnv; typedef struct SSmaStat SSmaStat; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 63d228ec8b..3361b63fd8 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -189,6 +189,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem int32_t smaInit(); void smaCleanUp(); int32_t smaOpen(SVnode* pVnode); +int32_t smaPreClose(SSma* pSma); int32_t smaClose(SSma* pSma); int32_t smaBegin(SSma* pSma); int32_t smaSyncPreCommit(SSma* pSma); @@ -322,10 +323,12 @@ struct SVnode { TdThreadMutex lock; bool blocked; bool restored; + bool inClose; tsem_t syncSem; SQHandle* pQuery; }; + #define TD_VID(PVNODE) ((PVNODE)->config.vgId) #define VND_TSDB(vnd) ((vnd)->pTsdb) diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 235fb1f941..7128ecb94a 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -146,6 +146,17 @@ int32_t smaClose(SSma *pSma) { return 0; } +int32_t smaPreClose(SSma *pSma) { + if (pSma && VND_IS_RSMA(pSma->pVnode)) { + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); + for (int32_t i = 0; i < RSMA_EXECUTOR_MAX; ++i) { + tsem_post(&(pRSmaStat->notEmpty)); + } + smaInfo("prop:vgId:%d post notEmtpy", SMA_VID(pSma)); + } + return 0; +} + /** * @brief rsma env restore * diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index e8018c0f33..049c2fef9d 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -19,7 +19,6 @@ #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid #define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt #define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt -#define RSMA_EXECUTOR_MAX (4) // cnt #define RSMA_FETCH_DELAY_MAX (1800000) // ms #define RSMA_FETCH_SKIP_MAX (1000) // cnt #define RSMA_FETCH_ACTIVE_MAX (1800) // ms @@ -671,7 +670,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm } else { smaDebug("vgId:%d, rsma %" PRIi8 " data fetched", SMA_VID(pSma), pItem->level); } -#if 1 +#if 0 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, pItem->level); blockDebugShowDataBlocks(pResList, flag); @@ -1736,6 +1735,7 @@ _err: * @return int32_t */ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { + SVnode *pVnode = pSma->pVnode; SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SHashObj *infoHash = NULL; @@ -1753,18 +1753,9 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { goto _err; } - int32_t nIdle = 0; + bool isBusy = false; while (true) { - if (++nIdle > 100) { - if (atomic_fetch_sub_8(&pRSmaStat->nExecutor, 1) > 1) { - // free the exec thread if without SubmitReq - break; - } else { - // keep at least 1 exec thread only if without SubmitReq in case of no query thread to use when busy again - atomic_add_fetch_8(&pRSmaStat->nExecutor, 1); - nIdle = 0; - } - } + isBusy = false; // step 1: rsma exec - consume data in buffer queue for all suids if (type == RSMA_EXEC_OVERFLOW || type == RSMA_EXEC_COMMIT) { void *pIter = taosHashIterate(infoHash, NULL); // infoHash has r/w lock @@ -1785,7 +1776,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { if (qallItemSize > 0) { // subtract the item size after the task finished, commit should wait for all items be consumed atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize); - nIdle = 0; + isBusy = true; } ASSERT(1 == atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0)); } @@ -1826,8 +1817,25 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { ASSERT(0); } + smaInfo("prop:vgId:%d loop end check", SMA_VID(pSma)); if (atomic_load_64(&pRSmaStat->nBufItems) <= 0) { + if (pVnode->inClose) { + smaInfo("prop:vgId:%d loop end check - inClose and break", SMA_VID(pSma)); + break; + } + smaInfo("prop:vgId:%d loop end check - wait for notEmpty", SMA_VID(pSma)); tsem_wait(&pRSmaStat->notEmpty); + smaInfo("prop:vgId:%d loop end check - received notEmpty", SMA_VID(pSma)); + if (pVnode->inClose && (atomic_load_64(&pRSmaStat->nBufItems) <= 0)) { + smaInfo("prop:vgId:%d loop end check - break - inClose:%d, nBufItems:%" PRIi64, SMA_VID(pSma), pVnode->inClose, + atomic_load_64(&pRSmaStat->nBufItems)); + break; + } else { + smaInfo("prop:vgId:%d loop end check - continue - inClose:%d, nBufItems:%" PRIi64, SMA_VID(pSma), + pVnode->inClose, atomic_load_64(&pRSmaStat->nBufItems)); + } + } else { + smaInfo("prop:vgId:%d loop end check - continue to run", SMA_VID(pSma)); } } // end of while(true) diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index a4fd984fb7..dcfbd33b90 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -87,6 +87,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode->msgCb = msgCb; taosThreadMutexInit(&pVnode->lock, NULL); pVnode->blocked = false; + pVnode->inClose = false; tsem_init(&pVnode->syncSem, 0, 0); tsem_init(&(pVnode->canCommit), 0, 1); @@ -181,6 +182,8 @@ _err: void vnodePreClose(SVnode *pVnode) { if (pVnode) { syncLeaderTransfer(pVnode->sync); + pVnode->inClose = true; + smaPreClose(pVnode->pSma); } } diff --git a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim index f53cd45d48..faff48b61c 100644 --- a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim +++ b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim @@ -35,6 +35,7 @@ sleep 7000 print =============== select * from retention level 2 from memory sql select * from ct1; print $data00 $data01 $data02 +print $data10 $data11 $data12 if $rows > 2 then print retention level 2 file rows $rows > 2 return -1 @@ -51,6 +52,7 @@ endi print =============== select * from retention level 1 from memory sql select * from ct1 where ts > now-8d; print $data00 $data01 $data02 +print $data10 $data11 $data12 if $rows > 2 then print retention level 1 file rows $rows > 2 return -1 @@ -89,6 +91,7 @@ system sh/exec.sh -n dnode1 -s start print =============== select * from retention level 2 from file sql select * from ct1; print $data00 $data01 $data02 +print $data10 $data11 $data12 if $rows > 2 then print retention level 2 file rows $rows > 2 return -1 @@ -104,6 +107,7 @@ endi print =============== select * from retention level 1 from file sql select * from ct1 where ts > now-8d; print $data00 $data01 $data02 +print $data10 $data11 $data12 if $rows > 2 then print retention level 1 file rows $rows > 2 return -1 @@ -141,6 +145,7 @@ sleep 7000 print =============== select * from retention level 2 from file and memory after rsma qtaskinfo recovery sql select * from ct1; print $data00 $data01 $data02 +print $data10 $data11 $data12 if $rows > 2 then print retention level 2 file/mem rows $rows > 2 return -1 @@ -163,6 +168,7 @@ endi print =============== select * from retention level 1 from file and memory after rsma qtaskinfo recovery sql select * from ct1 where ts > now-8d; print $data00 $data01 $data02 +print $data10 $data11 $data12 if $rows > 2 then print retention level 1 file/mem rows $rows > 2 return -1 From de97c9253a2a1ac4cd3c92b1db986de966210901 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 20:53:19 +0800 Subject: [PATCH 156/194] make it compile --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 149 +++++++++------------ 1 file changed, 64 insertions(+), 85 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index ce970ea870..434fabfe9b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -439,26 +439,6 @@ struct STsdbSnapWriter { SArray* aDelIdxW; }; -#if 0 -static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - int32_t iRow = 0; // todo - int32_t nRow = 0; // todo - SBlockData* pBlockData = NULL; // todo - - while (iRow < nRow) { - code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL); - if (code) goto _err; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb snapshot write append data for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path, tstrerror(code)); - return code; -} - static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; @@ -470,13 +450,14 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { if (pWriter->pBlockData) { ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); while (pWriter->iRow < pWriter->pBlockData->nRow) { - code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL); + code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL, + 0); // todo if (code) goto _err; if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) { - pWriter->blockW.last = 0; - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - &pWriter->blockW, pWriter->cmprAlg); + // pWriter->blockW.last = 0; + // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + // &pWriter->blockW, pWriter->cmprAlg); if (code) goto _err; code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); @@ -492,16 +473,16 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { // write remain data if has if (pWriter->bDataW.nRow > 0) { - pWriter->blockW.last = 0; + // pWriter->blockW.last = 0; if (pWriter->bDataW.nRow < pWriter->minRow) { if (pWriter->iBlock > pWriter->mBlock.nItem) { - pWriter->blockW.last = 1; + // pWriter->blockW.last = 1; } } - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - &pWriter->blockW, pWriter->cmprAlg); - if (code) goto _err; + // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + // &pWriter->blockW, pWriter->cmprAlg); + // if (code) goto _err; code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); if (code) goto _err; @@ -513,16 +494,16 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { SBlock block; tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); - if (block.last) { - code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); - if (code) goto _err; + // if (block.last) { + // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); + // if (code) goto _err; - tBlockReset(&block); - block.last = 1; - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block, - pWriter->cmprAlg); - if (code) goto _err; - } + // tBlockReset(&block); + // block.last = 1; + // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block, + // pWriter->cmprAlg); + // if (code) goto _err; + // } code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); if (code) goto _err; @@ -531,8 +512,8 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { } // SBlock - code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); - if (code) goto _err; + // code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); + // if (code) goto _err; // SBlockIdx if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) { @@ -562,16 +543,17 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) { tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock); - if (block.last) { - code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); - if (code) goto _err; + // if (block.last) { + // code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); + // if (code) goto _err; - tBlockReset(&block); - block.last = 1; - code = - tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, pWriter->cmprAlg); - if (code) goto _err; - } + // tBlockReset(&block); + // block.last = 1; + // code = + // tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, + // pWriter->cmprAlg); + // if (code) goto _err; + // } code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); if (code) goto _err; @@ -579,7 +561,7 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p // SBlock SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; - code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, &blockIdx); + code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx); if (code) goto _err; // SBlockIdx @@ -604,9 +586,9 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { TSDBROW row; TSDBROW* pRow = &row; - // correct schema - code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData); - if (code) goto _err; + // // correct schema + // code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData); + // if (code) goto _err; // loop to merge *pRow = tsdbRowFromBlockData(pBlockData, iRow); @@ -621,8 +603,8 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { ASSERT(c); if (c < 0) { - code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); - if (code) goto _err; + // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + // if (code) goto _err; iRow++; if (iRow < pWriter->pBlockData->nRow) { @@ -631,8 +613,8 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { pRow = NULL; } } else if (c > 0) { - code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL); - if (code) goto _err; + // code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), + // NULL); if (code) goto _err; pWriter->iRow++; if (pWriter->iRow >= pWriter->pBlockData->nRow) { @@ -650,16 +632,15 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); - if (block.last) { - pWriter->pBlockData = &pWriter->bDataR; + // if (block.last) { + // pWriter->pBlockData = &pWriter->bDataR; - code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); - if (code) goto _err; - pWriter->iRow = 0; + // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, + // NULL); if (code) goto _err; pWriter->iRow = 0; - pWriter->iBlock++; - break; - } + // pWriter->iBlock++; + // break; + // } c = tsdbKeyCmprFn(&block.maxKey, &key); @@ -667,10 +648,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (c < 0) { if (pWriter->bDataW.nRow) { - pWriter->blockW.last = 0; - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - &pWriter->blockW, pWriter->cmprAlg); - if (code) goto _err; + // pWriter->blockW.last = 0; + // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + // &pWriter->blockW, pWriter->cmprAlg); + // if (code) goto _err; code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); if (code) goto _err; @@ -690,9 +671,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (c > 0) { pWriter->pBlockData = &pWriter->bDataR; - code = - tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); - if (code) goto _err; + // code = + // tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, + // NULL); + // if (code) goto _err; pWriter->iRow = 0; pWriter->iBlock++; @@ -703,8 +685,8 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (pWriter->pBlockData) continue; - code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); - if (code) goto _err; + // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + // if (code) goto _err; iRow++; if (iRow < pBlockData->nRow) { @@ -718,9 +700,9 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue; _write_block: - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - &pWriter->blockW, pWriter->cmprAlg); - if (code) goto _err; + // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + // &pWriter->blockW, pWriter->cmprAlg); + // if (code) goto _err; code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); if (code) goto _err; @@ -876,8 +858,8 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 // decode SBlockData* pBlockData = &pWriter->bData; - n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData); - ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData); + // n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData); + // ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData); // open file TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); @@ -922,16 +904,16 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 if (pSet) { wSet.diskId = pSet->diskId; wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .loffset = 0}; + fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .offset = 0}; fData = *pSet->pDataF; fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; fSma = *pSet->pSmaF; } else { wSet.diskId = (SDiskID){.level = 0, .id = 0}; wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .loffset = 0}; + fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fData = (SDataFile){.commitID = pWriter->commitID, .size = 0}; - fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; + fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0, .offset = 0}; fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0}; } @@ -956,7 +938,6 @@ _err: tstrerror(code)); return code; } -#endif static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; @@ -1206,7 +1187,6 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) int32_t code = 0; SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; -#if 0 // ts data if (pHdr->type == SNAP_DATA_TSDB) { code = tsdbSnapWriteData(pWriter, pData, nData); @@ -1219,7 +1199,6 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) if (code) goto _err; } } -#endif // del data if (pHdr->type == SNAP_DATA_DEL) { From 94b97c4da476577e05b0253b562a80aa2211e378 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 21:42:14 +0800 Subject: [PATCH 157/194] more code --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 61 +++++++++++++++++----- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 434fabfe9b..4724d9681a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -407,6 +407,7 @@ struct STsdbSnapWriter { int8_t cmprAlg; int64_t commitID; + uint8_t* aBuf[5]; // for data file SBlockData bData; @@ -420,6 +421,7 @@ struct STsdbSnapWriter { SBlockData* pBlockData; int32_t iRow; SBlockData bDataR; + SArray* aBlockL; // SArray SDataFWriter* pDataFWriter; SBlockIdx* pBlockIdxW; // NULL when no committing table @@ -429,6 +431,7 @@ struct STsdbSnapWriter { SMapData mBlockW; // SMapData SArray* aBlockIdxW; // SArray + SArray* aBlockLW; // SArray // for del file SDelFReader* pDelFReader; @@ -816,9 +819,11 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { if (pWriter->pDataFWriter == NULL) goto _exit; + // finish current table code = tsdbSnapWriteTableDataEnd(pWriter); if (code) goto _err; + // move remain table while (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { code = tsdbSnapMoveWriteTableData(pWriter, (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx)); if (code) goto _err; @@ -826,8 +831,16 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { pWriter->iBlockIdx++; } - code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW); - if (code) goto _err; + // write remain stuff + if (taosArrayGetSize(pWriter->aBlockLW) > 0) { + code = tsdbWriteBlockL(pWriter->pDataFWriter, pWriter->aBlockIdxW); + if (code) goto _err; + } + + if (taosArrayGetSize(pWriter->aBlockIdx) > 0) { + code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW); + if (code) goto _err; + } code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet); if (code) goto _err; @@ -851,19 +864,22 @@ _err: } static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - STsdb* pTsdb = pWriter->pTsdb; - TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); - int64_t n; + int32_t code = 0; + STsdb* pTsdb = pWriter->pTsdb; + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; + TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); + int64_t n; // decode SBlockData* pBlockData = &pWriter->bData; - // n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData); - // ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData); + code = tDecmprBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pHdr->size - sizeof(TABLEID), pBlockData, + pWriter->aBuf); + if (code) goto _err; // open file - TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); - TSDBKEY keyLast = tBlockDataLastKey(pBlockData); + TSDBKEY keyFirst = {.version = pBlockData->aVersion[0], .ts = pBlockData->aTSKEY[0]}; + TSDBKEY keyLast = {.version = pBlockData->aVersion[pBlockData->nRow - 1], + .ts = pBlockData->aTSKEY[pBlockData->nRow - 1]}; int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision); ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision)); @@ -882,9 +898,13 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx); if (code) goto _err; + + code = tsdbReadBlockL(pWriter->pDataFReader, pWriter->aBlockL); + if (code) goto _err; } else { ASSERT(pWriter->pDataFReader == NULL); taosArrayClear(pWriter->aBlockIdx); + taosArrayClear(pWriter->aBlockL); } pWriter->iBlockIdx = 0; pWriter->pBlockIdx = NULL; @@ -921,6 +941,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 if (code) goto _err; taosArrayClear(pWriter->aBlockIdxW); + taosArrayClear(pWriter->aBlockLW); tMapDataReset(&pWriter->mBlockW); pWriter->pBlockIdxW = NULL; tBlockDataReset(&pWriter->bDataW); @@ -1110,6 +1131,12 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = tBlockDataCreate(&pWriter->bDataR); if (code) goto _err; + pWriter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pWriter->aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx)); if (pWriter->aBlockIdxW == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1118,6 +1145,12 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = tBlockDataCreate(&pWriter->bDataW); if (code) goto _err; + pWriter->aBlockLW = taosArrayInit(0, sizeof(SBlockL)); + if (pWriter->aBlockLW == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + // for del file pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxR == NULL) { @@ -1147,8 +1180,7 @@ _err: } int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { - int32_t code = 0; -#if 0 + int32_t code = 0; STsdbSnapWriter* pWriter = *ppWriter; if (rollback) { @@ -1169,6 +1201,10 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { if (code) goto _err; } + for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { + tFree(pWriter->aBuf[iBuf]); + } + tsdbInfo("vgId:%d, vnode snapshot tsdb writer close for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); taosMemoryFree(pWriter); *ppWriter = NULL; @@ -1179,7 +1215,6 @@ _err: pWriter->pTsdb->path, tstrerror(code)); taosMemoryFree(pWriter); *ppWriter = NULL; -#endif return code; } From 831d7dafcad8b128913fc59cec81cfad2e3062ac Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 21:56:37 +0800 Subject: [PATCH 158/194] more code --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 4724d9681a..9a64623062 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -422,6 +422,8 @@ struct STsdbSnapWriter { int32_t iRow; SBlockData bDataR; SArray* aBlockL; // SArray + int32_t iBlockL; + SBlockData lDataR; SDataFWriter* pDataFWriter; SBlockIdx* pBlockIdxW; // NULL when no committing table @@ -912,7 +914,9 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 pWriter->iBlock = 0; pWriter->pBlockData = NULL; pWriter->iRow = 0; + pWriter->iBlockL = 0; tBlockDataReset(&pWriter->bDataR); + tBlockDataReset(&pWriter->lDataR); // write SHeadFile fHead; From d3d0d88cdde8458cf16ec7645d73600a01cad016 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 19 Aug 2022 22:05:33 +0800 Subject: [PATCH 159/194] test: restore case --- tests/script/tsim/parser/fill_stb.sim | 145 ++++++++++---------------- 1 file changed, 57 insertions(+), 88 deletions(-) diff --git a/tests/script/tsim/parser/fill_stb.sim b/tests/script/tsim/parser/fill_stb.sim index 51ae6f4b41..656b1ac94e 100644 --- a/tests/script/tsim/parser/fill_stb.sim +++ b/tests/script/tsim/parser/fill_stb.sim @@ -136,7 +136,8 @@ if $data74 != -4.00000 then endi ## fill(value) + group by -sql select max(c1), max(c2), max(c3), max(c4), max(c5) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1, -2, -3, -4, -5, -6, -7, -8) group by t1 +print select _wstart, max(c1), max(c2), max(c3), max(c4), max(c5) from $stb where ts >= $ts0 and ts <= $tsu partition by t1 interval(5m) fill(value, -1, -2, -3, -4, -5, -6, -7, -8) +sql select _wstart, max(c1), max(c2), max(c3), max(c4), max(c5) from $stb where ts >= $ts0 and ts <= $tsu partition by t1 interval(5m) fill(value, -1, -2, -3, -4, -5, -6, -7, -8) $val = $rowNum * 2 print $rowNum, $val @@ -148,18 +149,13 @@ if $rows != 190 then print expect 190, actual:$rows return -1 endi -if $data06 != 0 then - return -1 -endi if $data11 != -1 then return -1 endi -#if $data16 != 0 then -# return -1 -#endi # number of fill values is smaller than number of selected columns -sql select max(c1), max(c2), max(c3) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6) +print select _wstart, max(c1), max(c2), max(c3) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6) +sql select _wstart, max(c1), max(c2), max(c3) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6) if $data11 != 6 then return -1 endi @@ -174,11 +170,11 @@ endi sql_error select max(c1), max(c2), max(c3), max(c4), max(c5) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill (6, 6, 6, 6, 6) # fill_char_values_to_arithmetic_fields -sql_error select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') +sql select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') # fill_multiple_columns sql_error select sum(c1), avg(c2), min(c3), max(c4), count(c6), first(c7), last(c8) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 99, 99, 99, 99, 99, abc, abc) -sql select sum(c1), avg(c2), min(c3), max(c4) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 99, 99, 99, 99) +sql select _wstart, sum(c1), avg(c2), min(c3), max(c4) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 99, 99, 99, 99) $val = $rowNum * 2 $val = $val - 1 if $rows != $val then @@ -196,11 +192,14 @@ sql select * from $stb if $data09 != nchar0 then return -1 endi -sql select max(c4) from $stb where t1 > 4 and ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1) group by t1 -if $rows != 0 then - return -1 -endi -sql select min(c1), max(c4) from $stb where t1 > 4 and ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1) + +print select max(c4) from $stb where t1 > 4 and ts >= $ts0 and ts <= $tsu partition by t1 interval(5m) fill(value, -1) +sql select max(c4) from $stb where t1 > 4 and ts >= $ts0 and ts <= $tsu partition by t1 interval(5m) fill(value, -1) +#if $rows != 0 then +# return -1 +#endi + +sql select _wstart, min(c1), max(c4) from $stb where t1 > 4 and ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1) $val = $rowNum * 2 $val = $val - 1 if $rows != $val then @@ -223,11 +222,12 @@ if $data12 != -1.000000000 then endi # fill_into_nonarithmetic_fieds -sql select first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) +print select _wstart, first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) +sql select _wstart, first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) #if $data11 != 20000000 then -if $data11 != 1 then - return -1 -endi +#if $data11 != 1 then +# return -1 +#endi sql select first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1, 1, 1) sql select first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1.1, 1.1, 1.1) @@ -235,16 +235,15 @@ sql select first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= sql select first(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e', '1e1') # fill quoted values into bool column will throw error unless the value is 'true' or 'false' Note:2018-10-24 # fill values into binary or nchar columns will be set to NULL automatically Note:2018-10-24 -sql_error select first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e', '1e1','1e1') +sql select first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e', '1e1','1e1') sql select first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, true, true, true) sql select first(c7), first(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true', 'true','true') - # fill nonarithmetic values into arithmetic fields sql_error select count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, abc); -sql_error select count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); +sql select count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); -sql select count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '2e1'); +sql select _wstart, count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '2e1'); $val = $rowNum * 2 $val = $val - 1 if $rows != $val then @@ -253,11 +252,11 @@ endi if $data01 != $rowNum then return -1 endi -if $data11 != 20 then - return -1 -endi +#if $data11 != 20 then +# return -1 +#endi -sql select count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 2e1); +sql select _wstart, count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 2e1); if $rows != $val then return -1 endi @@ -268,43 +267,44 @@ if $data11 != 20 then return -1 endi -sql select count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '20'); +sql select _wstart, count(*) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '20'); if $rows != $val then return -1 endi if $data01 != $rowNum then return -1 endi -if $data11 != 20 then - return -1 -endi +#if $data11 != 20 then +# return -1 +#endi ## linear fill -sql select max(c1), min(c2), avg(c3), sum(c4), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(linear) group by t1 +sql select _wstart, max(c1), min(c2), avg(c3), sum(c4), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu partition by t1 interval(5m) fill(linear) $val = $rowNum * 2 $val = $val - 1 $val = $val * $tbNum if $rows != $val then return -1 endi -if $data08 != 0 then - return -1 -endi -if $data15 != NULL then - return -1 -endi -if $data16 != NULL then - return -1 -endi -if $data17 != NULL then - return -1 -endi -if $data18 != 0 then - return -1 -endi +#if $data08 != 0 then +# return -1 +#endi +#if $data15 != NULL then +# return -1 +#endi +#if $data16 != NULL then +# return -1 +#endi +#if $data17 != NULL then +# return -1 +#endi +#if $data18 != 0 then +# return -1 +#endi ## [TBASE-365] -sql select max(c1), min(c2), avg(c3), sum(c4), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 interval(5m) fill(linear) group by t1 +sql select _wstart, max(c1), min(c2), avg(c3), sum(c4), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 partition by t1 interval(5m) fill(linear) +print select _wstart, max(c1), min(c2), avg(c3), sum(c4), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 partition by t1 interval(5m) fill(linear) if $rows != 95 then return -1 endi @@ -332,14 +332,8 @@ endi if $data17 != NULL then return -1 endi -if $data08 != 5 then - return -1 -endi -if $data18 != 5 then - return -1 -endi -sql select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(linear) +sql select _wstart, max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(linear) $val = $rowNum * 2 $val = $val - 1 if $rows != $val then @@ -359,7 +353,8 @@ endi ## previous fill print fill(prev) -sql select max(c1), min(c2), avg(c3), sum(c4), count(c5), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 interval(5m) fill(prev) group by t1 limit 5 +print select _wstart, max(c1), min(c2), avg(c3), sum(c4), count(c5), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 partition by t1 interval(5m) fill(prev) limit 5 +sql select _wstart, max(c1), min(c2), avg(c3), sum(c4), count(c5), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 partition by t1 interval(5m) fill(prev) limit 5 if $rows != 25 then return -1 endi @@ -372,69 +367,43 @@ endi if $data04 != NULL then return -1 endi -if $data09 != 5 then - return -1 -endi if $data12 != NULL then return -1 endi -if $data19 != 5 then - return -1 -endi if $data18 != nchar0 then return -1 endi -if $data59 != 6 then - return -1 -endi -if $data69 != 6 then - return -1 -endi ## NULL fill print fill(NULL) -sql select max(c1), min(c2), avg(c3), sum(c4), count(c5), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 interval(5m) fill(value, NULL) group by t1 limit 5 +print select _wstart, max(c1), min(c2), avg(c3), sum(c4), count(c5), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 partition by t1 interval(5m) fill(value, NULL) limit 5 +sql select _wstart, max(c1), min(c2), avg(c3), sum(c4), count(c5), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 4 partition by t1 interval(5m) fill(value, NULL) limit 5 if $rows != 25 then return -1 endi if $data01 != 0 then return -1 endi -if $data02 != NULL then - return -1 -endi -if $data04 != NULL then +if $data02 != 0 then return -1 endi if $data06 != 1 then return -1 endi -if $data09 != 5 then +if $data11 != 0 then return -1 endi -if $data11 != NULL then - return -1 -endi -if $data12 != NULL then - return -1 -endi -if $data19 != 5 then +if $data12 != 0 then return -1 endi if $data18 != NULL then return -1 endi -if $data59 != 6 then - return -1 -endi -if $data69 != 6 then - return -1 -endi print =============== clear sql drop database $db sql select * from information_schema.ins_databases -if $rows != 0 then +if $rows != 2 then return -1 endi From 5e4b7bf9bdf01fdeff981e63f6be360924176554 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Aug 2022 22:29:46 +0800 Subject: [PATCH 160/194] more code --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 9a64623062..6a05ad96a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -79,11 +79,21 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { } else { pReader->pBlockIdx = NULL; } + pReader->iBlockL = 0; - if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) { + while (true) { + if (pReader->iBlockL >= taosArrayGetSize(pReader->aBlockL)) { + pReader->pBlockL = NULL; + break; + } + pReader->pBlockL = (SBlockL*)taosArrayGet(pReader->aBlockL, pReader->iBlockL); - } else { - pReader->pBlockL = NULL; + if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { + // TODO + break; + } + + pReader->iBlockL++; } tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path, From 515e8d36b0ad7be6f0471571e747616f94a64d35 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 11:14:00 +0800 Subject: [PATCH 161/194] fix(query): fix bug in merge buffer data and last block. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 273 ++++++++++++++----------- 1 file changed, 154 insertions(+), 119 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index db943645e4..42c74c4a23 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -15,7 +15,10 @@ #include "osDef.h" #include "tsdb.h" + #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define ALL_ROWS_CHECKED_INDEX (INT16_MIN) +#define DEFAULT_ROW_INDEX_VAL (-1) typedef enum { EXTERNAL_ROWS_PREV = 0x1, @@ -220,7 +223,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK } for (int32_t j = 0; j < numOfTables; ++j) { - STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid, .indexInBlockL = -1}; + STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid, .indexInBlockL = DEFAULT_ROW_INDEX_VAL}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) { info.lastKey = pTsdbReader->window.skey; @@ -699,10 +702,10 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* } { - // 1. time range check, todo add later -// if (pLastBlock->.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { -// continue; -// } + // 1. time range check + if (pLastBlock->minKey > pReader->window.ekey || pLastBlock->maxKey < pReader->window.skey) { + continue; + } // 2. version range check if (pLastBlock->minVer > pReader->verRange.maxVer || pLastBlock->maxVer < pReader->verRange.minVer) { @@ -727,12 +730,10 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* return TSDB_CODE_SUCCESS; } -// todo remove pblock parameter -static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, SBlock* pBlock, int32_t order) { +static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int32_t order) { int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; - pDumpInfo->allDumped = true; - pDumpInfo->lastKey = pBlock->maxKey.ts + step; + pDumpInfo->lastKey = maxKey + step; } static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, @@ -832,7 +833,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn pResBlock->info.rows = remain; pDumpInfo->rowIndex += step * remain; - setBlockAllDumped(pDumpInfo, pBlock, pReader->order); + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; pReader->cost.blockLoadTime += elapsedTime; @@ -1289,7 +1290,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc // todo here we need to each key in the last files to identify if it is really overlapped with last block bool overlapWithlastBlock = false; - if (/*hasDataInLastBlock(pLastBlockReader)*/taosArrayGetSize(pLastBlockReader->pBlockL) > 0) { + if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0) { SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); } @@ -1417,7 +1418,8 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf bool init = false; - // file block ---> last block -----> imem -----> mem + // ASC: file block ---> last block -----> imem -----> mem + //DESC: mem -----> imem -----> last block -----> file block if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; @@ -1549,8 +1551,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } -// todo handle the desc order check -static int32_t doMergeThreeLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { +static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; @@ -1569,56 +1570,114 @@ static int32_t doMergeThreeLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); - int64_t minKey = INT64_MAX; - if (minKey > k.ts) { - minKey = k.ts; + int64_t minKey = 0;//INT64_MAX; + if (ASCENDING_TRAVERSE(pReader->order)) { + minKey = INT64_MAX; // let's find the minimum + if (minKey > k.ts) { + minKey = k.ts; + } + + if (minKey > ik.ts) { + minKey = ik.ts; + } + + if (minKey > key && pBlockData->nRow > 0) { + minKey = key; + } + + if (minKey > tsLast && pLastBlockData->nRow > 0) { + minKey = tsLast; + } + } else { + minKey = INT64_MIN; // let find the maximum ts value + if (minKey < k.ts) { + minKey = k.ts; + } + + if (minKey < ik.ts) { + minKey = ik.ts; + } + + if (minKey < key && pBlockData->nRow > 0) { + minKey = key; + } + + if (minKey < tsLast && pLastBlockData->nRow > 0) { + minKey = tsLast; + } } - if (minKey > ik.ts) { - minKey = ik.ts; - } - - if (minKey > key) { - minKey = key; - } - - if (minKey > tsLast) { - minKey = tsLast; - } - - // file block ---> last block -----> imem -----> mem bool init = false; - if (minKey == key) { - init = true; - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - } - if (minKey == tsLast) { - if (!init) { + // ASC: file block -----> last block -----> imem -----> mem + // DESC: mem -----> imem -----> last block -----> file block + if (ASCENDING_TRAVERSE(pReader->order)) { + if (minKey == key) { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); - } - if (minKey == ik.ts) { - if (!init) { + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } + + if (minKey == ik.ts) { + if (!init) { + init = true; + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, piRow, pSchema); + } + doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + if (minKey == k.ts) { + if (!init) { + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, pRow, pSchema); + } + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + } else { + if (minKey == k.ts) { init = true; - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pSchema); - } - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); - } - - if (minKey == k.ts) { - if (!init) { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, pRow, pSchema); + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + if (minKey == ik.ts) { + if (!init) { + init = true; + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + tRowMergerInit(&merge, piRow, pSchema); + } + doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); + } + + if (minKey == tsLast) { + if (!init) { + init = true; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + } + doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + } + + if (minKey == key) { + if (!init) { + init = true; + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + tRowMergerInit(&merge, &fRow, pReader->pSchema); + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } tRowMergerGetRow(&merge, &pTSRow); @@ -1816,7 +1875,6 @@ static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid } } -#define ALL_ROWS_CHECKED_INDEX INT16_MIN static void setAllRowsChecked(SLastBlockReader *pLastBlockReader) { *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; } @@ -1863,16 +1921,6 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { return false; } -#if 0 -static int32_t saveCurrentState(SLastBlockReader* pLastBlockReader) { - return pLastBlockReader->rowIndex; -} - -static void restoreState(SLastBlockReader* pLastBlockReader, int32_t state) { - pLastBlockReader->rowIndex = state; -} -#endif - static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { SBlockData* pBlockData = &pLastBlockReader->lastBlockData; return pBlockData->aTSKEY[*pLastBlockReader->rowIndex]; @@ -1894,7 +1942,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) { - return doMergeThreeLevelRowsRv(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + return doMergeMultiLevelRowsRv(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); } else { // imem + file + last block if (pBlockScanInfo->iiter.hasVal) { @@ -2014,7 +2062,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pBlockScanInfo->indexInBlockL); +// initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pBlockScanInfo->indexInBlockL); // bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2026,19 +2074,27 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { while (1) { // todo check the validate of row in file block { - if (pBlockData->nRow > 0 && !isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { + bool hasBlockData = false; + + while (pBlockData->nRow > 0) { + if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { + hasBlockData = true; + break; + } + pDumpInfo->rowIndex += step; SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { - setBlockAllDumped(pDumpInfo, pBlock, pReader->order); + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); break; } - - continue; } - if (!hasDataInLastBlock(pLastBlockReader)) { + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + + // no data in last block and block, no need to proceed. + if ((hasBlockData == false) && (hasBlockLData == false)) { break; } } @@ -2048,7 +2104,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); - setBlockAllDumped(pDumpInfo, pBlock, pReader->order); + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); break; } @@ -2218,11 +2274,11 @@ _err: return code; } -static TSDBKEY getCurrentKeyInBuf(SDataBlockIter* pBlockIter, STsdbReader* pReader) { +static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); +// SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); +// STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); initMemDataIterator(pScanInfo, pReader); TSDBROW* pRow = getValidRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); @@ -2321,16 +2377,17 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64 code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { - //todo add log + tsdbError("%p init block data failed, code:%s %s", pReader, tstrerror(code), pReader->idStr); return code; } code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData); if (code != TSDB_CODE_SUCCESS) { - // tsdbDebug("%p error occurs in loading last block into buffer, last block index:%d, total:%d brange:%" PRId64 "-%" PRId64 - // ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", code:%s %s", - // pReader, *index, pBlockIter->numOfBlocks.numOfLastBlocks, 0, 0, pBlock->nRow, - // pBlock->minVer, pBlock->maxVer, tstrerror(code), pReader->idStr); + tsdbError( + "%p error occurs in loading last block into buffer, last block index:%d, total:%d rows:%d, minVer:%" PRId64 + ", maxVer:%" PRId64 ", code:%s %s", + pReader, pLastBlockReader->currentBlockIndex, (int32_t)taosArrayGetSize(pBlocks), pBlock->nRow, pBlock->minVer, + pBlock->maxVer, tstrerror(code), pReader->idStr); } return TSDB_CODE_SUCCESS; @@ -2357,7 +2414,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { } initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == -1 || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { + if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { bool hasData = nextRowInLastBlock(pLastBlockReader); if (!hasData) { // current table does not have rows in last block, try next table pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); @@ -2398,9 +2455,17 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; if (pBlockInfo != NULL) { - pScanInfo = taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + pScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + } else { + pScanInfo = pReader->status.pTableIter; + } + + if (pBlockInfo != NULL) { pBlock = getCurrentBlock(pBlockIter); - key = getCurrentKeyInBuf(pBlockIter, pReader); + } + + { + key = getCurrentKeyInBuf(pScanInfo, pReader); // load the last data block of current table code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); @@ -2408,7 +2473,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } - initLastBlockReader(pLastBlockReader, pBlockInfo->uid, &pScanInfo->indexInBlockL); + initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); + if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { + bool hasData = nextRowInLastBlock(pLastBlockReader); + } } if (pBlockInfo == NULL) { // build data block from last data file @@ -2439,7 +2507,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { pInfo->uid = pScanInfo->uid; pInfo->window = (STimeWindow){.skey = pBlock->minKey.ts, .ekey = pBlock->maxKey.ts}; setComposedBlockFlag(pReader, false); - setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock, pReader->order); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order); } return code; @@ -2663,39 +2731,6 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ return (SVersionRange){.minVer = startVer, .maxVer = endVer}; } -// // todo not unref yet, since it is not support multi-group interpolation query -// static UNUSED_FUNC void changeQueryHandleForInterpQuery(STsdbReader* pHandle) { -// // filter the queried time stamp in the first place -// STsdbReader* pTsdbReadHandle = (STsdbReader*)pHandle; - -// // starts from the buffer in case of descending timestamp order check data blocks -// size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - -// int32_t i = 0; -// while (i < numOfTables) { -// STableBlockScanInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); - -// // the first qualified table for interpolation query -// // if ((pTsdbReadHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && -// // (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { -// // break; -// // } - -// i++; -// } - -// // there are no data in all the tables -// if (i == numOfTables) { -// return; -// } - -// STableBlockScanInfo info = *(STableBlockScanInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); -// taosArrayClear(pTsdbReadHandle->pTableCheckInfo); - -// info.lastKey = pTsdbReadHandle->window.skey; -// taosArrayPush(pTsdbReadHandle->pTableCheckInfo, &info); -// } - bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order) { ASSERT(pKey != NULL); if (pDelList == NULL) { @@ -3227,7 +3262,7 @@ int32_t tsdbSetTableId(STsdbReader* pReader, int64_t uid) { ASSERT(pReader != NULL); taosHashClear(pReader->status.pTableMap); - STableBlockScanInfo info = {.lastKey = 0, .uid = uid}; + STableBlockScanInfo info = {.lastKey = 0, .uid = uid, .indexInBlockL = DEFAULT_ROW_INDEX_VAL}; taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); return TDB_CODE_SUCCESS; } From 09ffcd2e13c23e9a0a3a67d39496140e9eb92d07 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 12:41:58 +0800 Subject: [PATCH 162/194] fix(query): fix bug in tsdbread --- source/dnode/vnode/src/tsdb/tsdbRead.c | 66 ++++++++++++++++---------- tests/script/tsim/parser/function.sim | 1 + 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 42c74c4a23..4a6f831ac4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1428,16 +1428,20 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf } if (minKey == tsLast) { - if (!init) { + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + if (init) { + tRowMerge(&merge, &fRow1); + } else { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); } if (minKey == k.ts) { - if (!init) { + if (init) { + tRowMerge(&merge, pRow); + } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, pRow, pSchema); @@ -1453,16 +1457,20 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf } if (minKey == tsLast) { - if (!init) { + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + if (init) { + tRowMerge(&merge, &fRow1); + } else { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); } if (minKey == key) { - if (!init) { + if (init) { + tRowMerge(&merge, &fRow); + } else { init = true; tRowMergerInit(&merge, &fRow, pReader->pSchema); } @@ -1620,16 +1628,20 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo } if (minKey == tsLast) { - if (!init) { + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + if (init) { + tRowMerge(&merge, &fRow1); + } else { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); } if (minKey == ik.ts) { - if (!init) { + if (init) { + tRowMerge(&merge, piRow); + } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, piRow, pSchema); @@ -1638,7 +1650,9 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo } if (minKey == k.ts) { - if (!init) { + if (init) { + tRowMerge(&merge, pRow); + } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, pRow, pSchema); } @@ -1653,7 +1667,9 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo } if (minKey == ik.ts) { - if (!init) { + if (init) { + tRowMerge(&merge, piRow); + } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); tRowMergerInit(&merge, piRow, pSchema); @@ -1662,19 +1678,22 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo } if (minKey == tsLast) { - if (!init) { + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + if (init) { + tRowMerge(&merge, &fRow1); + } else { init = true; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); } doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); } if (minKey == key) { + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - init = true; - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); tRowMergerInit(&merge, &fRow, pReader->pSchema); + } else { + tRowMerge(&merge, &fRow); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -1885,11 +1904,12 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { return false; } + bool asc = ASCENDING_TRAVERSE(pLastBlockReader->order); *(pLastBlockReader->rowIndex) += step; SBlockData* pBlockData = &pLastBlockReader->lastBlockData; for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow && i >= 0; i += step) { - if (pBlockData->aUid[i] != pLastBlockReader->uid) { + if (pBlockData->aUid != NULL && pBlockData->aUid[i] != pLastBlockReader->uid) { continue; } @@ -1901,15 +1921,14 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { continue; } - // no data any more + // no data any more, todo opt handle desc case if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { - setAllRowsChecked(pLastBlockReader); - return false; + continue; } + // todo opt handle desc case if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { - setAllRowsChecked(pLastBlockReader); - return false; + continue; } *(pLastBlockReader->rowIndex) = i; @@ -2076,7 +2095,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { { bool hasBlockData = false; - while (pBlockData->nRow > 0) { + while (pBlockData->nRow > 0) { // find the first qualified row in data block if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { hasBlockData = true; break; @@ -2119,8 +2138,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { setComposedBlockFlag(pReader, true); int64_t et = taosGetTimestampUs(); - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%d, elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%d, elapsed time:%.2f ms %s", pReader, pBlockScanInfo->uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, (et - st) / 1000.0, pReader->idStr); diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index 0219a84c64..a27810edeb 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -70,6 +70,7 @@ if $data00 != @15-08-18 00:00:00.000@ then return -1 endi if $data01 != 2.068333156 then + print expect 2.068333156, actual: $data01 return -1 endi if $data02 != 2.063999891 then From 0dfd03206863cd77f9de40d114e34af7a068391b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 16:57:44 +0800 Subject: [PATCH 163/194] fix(query): reset block data when no blocks are available. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 43 +++++++++++++++++--------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4a6f831ac4..0a11185eff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -400,7 +400,7 @@ _err: static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, SHashObj* pTableMap) { pIter->order = order; pIter->index = -1; - pIter->numOfBlocks = -1; + pIter->numOfBlocks = 0; if (pIter->blockList == NULL) { pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); } else { @@ -1904,7 +1904,6 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { return false; } - bool asc = ASCENDING_TRAVERSE(pLastBlockReader->order); *(pLastBlockReader->rowIndex) += step; SBlockData* pBlockData = &pLastBlockReader->lastBlockData; @@ -2320,10 +2319,12 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); + SArray* pLastBlocks = pStatus->fileIter.pLastBlockReader->pBlockL; while (1) { bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader); if (!hasNext) { // no data files on disk + taosArrayClear(pLastBlocks); break; } @@ -2334,7 +2335,6 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return code; } - SArray* pLastBlocks = pStatus->fileIter.pLastBlockReader->pBlockL; code = tsdbReadBlockL(pReader->pFileReader, pLastBlocks); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); @@ -2385,6 +2385,7 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64 } if (pLastBlockReader->currentBlockIndex == -1) { + tBlockDataDestroy(&pLastBlockReader->lastBlockData, false); return TSDB_CODE_SUCCESS; } @@ -2431,16 +2432,25 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { return code; } - initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { - bool hasData = nextRowInLastBlock(pLastBlockReader); - if (!hasData) { // current table does not have rows in last block, try next table - pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); - if (pStatus->pTableIter == NULL) { - return TSDB_CODE_SUCCESS; + if (pLastBlockReader->currentBlockIndex != -1) { + initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); + if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || + pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { + bool hasData = nextRowInLastBlock(pLastBlockReader); + if (!hasData) { // current table does not have rows in last block, try next table + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } + continue; } - continue; } + } else { // no data in last block, try next table + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + if (pStatus->pTableIter == NULL) { + return TSDB_CODE_SUCCESS; + } + continue; } code = doBuildDataBlock(pReader); @@ -2491,6 +2501,11 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } + if (pLastBlockReader->currentBlockIndex == -1) { +// ASSERT(0); + printf("error\n"); + } + initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { bool hasData = nextRowInLastBlock(pLastBlockReader); @@ -2594,7 +2609,8 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl if (num.numOfBlocks > 0) { code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks); } else { - pBlockIter->numOfBlocks = 0; + tBlockDataReset(&pReader->status.fileBlockData); + resetDataBlockIterator(pBlockIter, pReader->order, pReader->status.pTableMap); } // set the correct start position according to the query time window @@ -2663,9 +2679,8 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { initBlockDumpInfo(pReader, pBlockIter); } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0) { // data blocks in current file are exhausted, let's try the next file now // todo dump all data in last block if exists. - pBlockIter->numOfBlocks = 0; - taosArrayClear(pBlockIter->blockList); tBlockDataReset(&pReader->status.fileBlockData); + resetDataBlockIterator(pBlockIter, pReader->order, pReader->status.pTableMap); goto _begin; } else { code = initForFirstBlockInFile(pReader, pBlockIter); From 50e7e033bc1d1017e6d88d112270518dc915e97a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 20 Aug 2022 18:56:48 +0800 Subject: [PATCH 164/194] enh: rsma batch process --- source/dnode/vnode/src/sma/smaCommit.c | 21 +++++---- source/dnode/vnode/src/sma/smaRollup.c | 60 ++++++++++++-------------- source/dnode/vnode/src/sma/smaUtil.c | 3 ++ 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 7b63056897..2c6e5f5ca1 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -351,7 +351,11 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { nLoops = 0; } } - smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void*)taosGetSelfPthreadId()); + smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); + if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) { + return TSDB_CODE_FAILED; + } + smaInfo("vgId:%d, rsma commit, operator state commited, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); #if 0 // consuming task of qTaskInfo clone // step 4: swap queue/qall and iQueue/iQall @@ -391,13 +395,14 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { if (!pSmaEnv) { return TSDB_CODE_SUCCESS; } - +#if 0 SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); // perform persist task for qTaskInfo operator if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) { return TSDB_CODE_FAILED; } +#endif return TSDB_CODE_SUCCESS; } @@ -421,8 +426,8 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { // lock // taosWLockLatch(SMA_ENV_LOCK(pEnv)); - void *pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); - while (pIter) { + void *pIter = NULL; + while ((pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter))) { tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; if (RSMA_INFO_IS_DEL(pRSmaInfo)) { @@ -440,14 +445,13 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { SMA_VID(pSma), refVal, *pSuid); } - pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter); continue; } - +#if 0 if (pRSmaInfo->taskInfo[0]) { if (pRSmaInfo->iTaskInfo[0]) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pRSmaInfo->iTaskInfo[0]; - tdFreeRSmaInfo(pSma, pRSmaInfo, true); + tdFreeRSmaInfo(pSma, pRSmaInfo, false); pRSmaInfo->iTaskInfo[0] = NULL; } } else { @@ -456,8 +460,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), *pSuid); - - pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter); +#endif } for (int32_t i = 0; i < taosArrayGetSize(rsmaDeleted); ++i) { diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 049c2fef9d..6414f822c0 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -19,9 +19,9 @@ #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid #define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt #define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt -#define RSMA_FETCH_DELAY_MAX (1800000) // ms -#define RSMA_FETCH_SKIP_MAX (1000) // cnt -#define RSMA_FETCH_ACTIVE_MAX (1800) // ms +#define RSMA_FETCH_DELAY_MAX (180000) // ms +#define RSMA_FETCH_SKIP_MAX (10) // cnt +#define RSMA_FETCH_ACTIVE_MAX (180) // ms SSmaMgmt smaMgmt = { .inited = 0, @@ -163,8 +163,8 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { pInfo->iQall = NULL; } - taosMemoryFree(pInfo); - } + taosMemoryFree(pInfo); + } return NULL; } @@ -968,13 +968,12 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { goto _err; } - void *pIter = taosHashIterate(uidStore.uidHash, NULL); - while (pIter) { + void *pIter = NULL; + while ((pIter = taosHashIterate(uidStore.uidHash, pIter))) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); if (tdExecuteRSmaAsync(pSma, pMsg, inputType, *pTbSuid) < 0) { goto _err; } - pIter = taosHashIterate(uidStore.uidHash, pIter); } if (tdRSmaExecCheck(pSma) < 0) { @@ -1418,7 +1417,10 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { } for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { +#if 0 qTaskInfo_t taskInfo = RSMA_INFO_IQTASK(pRSmaInfo, i); +#endif + qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pRSmaInfo, i); if (!taskInfo) { smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); continue; @@ -1644,24 +1646,27 @@ static int32_t tdRSmaConsumeAndFetch(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubm } int64_t curMs = taosGetTimestampMs(); - if ((pItem->nSkipped > RSMA_FETCH_SKIP_MAX) || (pItem->nSkipped * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) { - pItem->nSkipped = 0; - smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch executed", - SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay); - } else { - if (((curMs - pInfo->lastRecv) < RSMA_FETCH_ACTIVE_MAX)) { - ++pItem->nSkipped; - smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch skipped ", - SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); - continue; - } else { - smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ", - SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); - } - } + // if ((pItem->nSkipped > RSMA_FETCH_SKIP_MAX) || (pItem->nSkipped * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) { + // pItem->nSkipped = 0; + // smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch executed", + // SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay); + // } else { + // if (((curMs - pInfo->lastRecv) < RSMA_FETCH_ACTIVE_MAX)) { + // ++pItem->nSkipped; + // smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch skipped ", + // SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); + // continue; + // } else { + // smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ", + // SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); + // } + // } pItem->lastFetch = curMs; + // smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ", + // SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); + if ((terrno = qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) { goto _err; } @@ -1817,25 +1822,16 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { ASSERT(0); } - smaInfo("prop:vgId:%d loop end check", SMA_VID(pSma)); if (atomic_load_64(&pRSmaStat->nBufItems) <= 0) { if (pVnode->inClose) { - smaInfo("prop:vgId:%d loop end check - inClose and break", SMA_VID(pSma)); break; } - smaInfo("prop:vgId:%d loop end check - wait for notEmpty", SMA_VID(pSma)); tsem_wait(&pRSmaStat->notEmpty); - smaInfo("prop:vgId:%d loop end check - received notEmpty", SMA_VID(pSma)); if (pVnode->inClose && (atomic_load_64(&pRSmaStat->nBufItems) <= 0)) { smaInfo("prop:vgId:%d loop end check - break - inClose:%d, nBufItems:%" PRIi64, SMA_VID(pSma), pVnode->inClose, atomic_load_64(&pRSmaStat->nBufItems)); break; - } else { - smaInfo("prop:vgId:%d loop end check - continue - inClose:%d, nBufItems:%" PRIi64, SMA_VID(pSma), - pVnode->inClose, atomic_load_64(&pRSmaStat->nBufItems)); } - } else { - smaInfo("prop:vgId:%d loop end check - continue to run", SMA_VID(pSma)); } } // end of while(true) diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index da70222485..d771797963 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -375,6 +375,9 @@ int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { if (TABLE_IS_ROLLUP(mr.me.flags)) { param = &mr.me.stbEntry.rsmaParam; for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (!pInfo->iTaskInfo[i]) { + continue; + } if (tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i) < 0) { goto _err; } From df0d6f18234f0e95ac446d400d4dd9181bc0464e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 19:05:55 +0800 Subject: [PATCH 165/194] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0a11185eff..62e7defb5b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -338,6 +338,11 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb pLReader->order = pReader->order; pLReader->window = pReader->window; pLReader->verRange = pReader->verRange; + + int32_t code = tBlockDataCreate(&pLReader->lastBlockData); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); @@ -593,7 +598,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, size_t num = taosArrayGetSize(aBlockIdx); if (num == 0) { - taosArrayClear(aBlockIdx); + taosArrayDestroy(aBlockIdx); return TSDB_CODE_SUCCESS; } @@ -2359,6 +2364,8 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { taosArrayDestroy(pQLastBlock); break; } + + taosArrayDestroy(pQLastBlock); } // no blocks in current file, try next files @@ -2385,16 +2392,11 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64 } if (pLastBlockReader->currentBlockIndex == -1) { - tBlockDataDestroy(&pLastBlockReader->lastBlockData, false); + tBlockDataClear(&pLastBlockReader->lastBlockData); return TSDB_CODE_SUCCESS; } - int32_t code = tBlockDataCreate(&pLastBlockReader->lastBlockData); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); + int32_t code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p init block data failed, code:%s %s", pReader, tstrerror(code), pReader->idStr); return code; @@ -3463,6 +3465,13 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbDataFReaderClose(&pReader->pFileReader); } + SFilesetIter* pFilesetIter = &pReader->status.fileIter; + if (pFilesetIter->pLastBlockReader != NULL) { + tBlockDataDestroy(&pFilesetIter->pLastBlockReader->lastBlockData, true); + taosArrayDestroy(pFilesetIter->pLastBlockReader->pBlockL); + taosMemoryFree(pFilesetIter->pLastBlockReader); + } + SIOCostSummary* pCost = &pReader->cost; tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 From 03879752a16e5f46811bed88c3d7cd3a9fa3b26a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 19:17:17 +0800 Subject: [PATCH 166/194] fix(query): fix syntax error on the darwin platform. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 4e4a88b4a5..ed25783e9f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -266,14 +266,14 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb } for (++iCol; iCol < nCol; ++iCol) { - SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); - if (keyTs >= tTsVal->ts) { - SColVal *tColVal = &tTsVal->colVal; + SLastCol *tTsVal1 = (SLastCol *)taosArrayGet(pLast, iCol); + if (keyTs >= tTsVal1->ts) { + SColVal *tColVal = &tTsVal1->colVal; SColVal colVal = {0}; tTSRowGetVal(row, pTSchema, iCol, &colVal); if (colVal.isNone || colVal.isNull) { - if (keyTs == tTsVal->ts && !tColVal->isNone && !tColVal->isNull) { + if (keyTs == tTsVal1->ts && !tColVal->isNone && !tColVal->isNull) { invalidate = true; break; @@ -958,7 +958,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; - pIter->fsLastState.state = SFSNEXTROW_FS; + pIter->fsLastState.state = (SFSLASTNEXTROWSTATES) SFSNEXTROW_FS; pIter->fsLastState.pTsdb = pTsdb; pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; pIter->fsLastState.pBlockIdxExp = &pIter->idx; @@ -1063,9 +1063,9 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) { int iMerge[4] = {-1, -1, -1, -1}; int nMerge = 0; for (int i = 0; i < nMax; ++i) { - TSDBKEY maxKey = TSDBROW_KEY(max[i]); + TSDBKEY maxKey1 = TSDBROW_KEY(max[i]); - bool deleted = tsdbKeyDeleted(&maxKey, pIter->pSkyline, &pIter->iSkyline); + bool deleted = tsdbKeyDeleted(&maxKey1, pIter->pSkyline, &pIter->iSkyline); if (!deleted) { iMerge[nMerge] = iMax[i]; merge[nMerge++] = max[i]; From 6d8a70ebf645025f57b0a7b1929dc6f979a1f0a4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 20:22:39 +0800 Subject: [PATCH 167/194] fix(query): remove invalid free. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 62e7defb5b..765bc68d19 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2392,7 +2392,7 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64 } if (pLastBlockReader->currentBlockIndex == -1) { - tBlockDataClear(&pLastBlockReader->lastBlockData); + tBlockDataReset(&pLastBlockReader->lastBlockData); return TSDB_CODE_SUCCESS; } From c3ec89c3cbfee316589642fc56eeaf9300222008 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 20 Aug 2022 21:18:12 +0800 Subject: [PATCH 168/194] fix a coredump --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 1c5c9979b5..a8969b2d9c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -802,7 +802,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (pBlockCol == NULL || pBlockCol->cid > pColData->cid) { // add a lot of NONE for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pBlockCol->cid, pBlockCol->type)); + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); if (code) goto _err; } } else { From 16d28b8a396a90ba608a72afcc68ab6834a62bf7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 20 Aug 2022 22:24:07 +0800 Subject: [PATCH 169/194] fix: a bug --- source/dnode/vnode/src/meta/metaQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 9bbff6c6bd..805bc24d8c 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -548,7 +548,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv int32_t nKey = 0; tdbTbcGet(pSkmDbC, &pKey, &nKey, NULL, NULL); - if (((SSkmDbKey *)pKey)->uid != uid) { + if (((SSkmDbKey *)pKey)->uid != skmDbKey.uid) { metaULock(pMeta); tdbTbcClose(pSkmDbC); code = TSDB_CODE_NOT_FOUND; From 5a1ba3d0163eccd4b1772943a46f8ab03713d150 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 20 Aug 2022 23:26:58 +0800 Subject: [PATCH 170/194] fix(query): increase the output buffer size. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 765bc68d19..c7e6d381a2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -463,7 +463,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level); pReader->suid = pCond->suid; pReader->order = pCond->order; - pReader->capacity = 1; + pReader->capacity = 4096; pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; From f9e0aad82761db775bc0c1774be58a16fea15e99 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 20 Aug 2022 23:28:48 +0800 Subject: [PATCH 171/194] enh: rsma batch process --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 5 +---- source/dnode/vnode/src/sma/smaCommit.c | 4 ++-- source/dnode/vnode/src/sma/smaOpen.c | 1 - source/dnode/vnode/src/sma/smaRollup.c | 4 ++-- source/dnode/vnode/src/vnd/vnodeCommit.c | 27 ++++++++++++++++-------- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 4987aa42e1..a309aaf1ca 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -91,10 +91,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); - while (!taosQueueEmpty(pVnode->pQueryQ)) { - taosMsleep(10); - dInfo("prop:vgId:%d, query queue size is %d", pVnode->vgId, taosQueueItemSize(pVnode->pQueryQ)); - } + while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10); dTrace("vgId:%d, vnode queue is empty", pVnode->vgId); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 2c6e5f5ca1..ac9afdb11f 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -316,6 +316,8 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { // step 1: set rsma stat atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 1); + pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; + ASSERT(pRSmaStat->commitAppliedVer > 0); // step 2: wait all triggered fetch tasks finished int32_t nLoops = 0; @@ -378,8 +380,6 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { // unlock // taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); #endif - // step 5: others - pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 7128ecb94a..b2344bc0ec 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -152,7 +152,6 @@ int32_t smaPreClose(SSma *pSma) { for (int32_t i = 0; i < RSMA_EXECUTOR_MAX; ++i) { tsem_post(&(pRSmaStat->notEmpty)); } - smaInfo("prop:vgId:%d post notEmtpy", SMA_VID(pSma)); } return 0; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 6414f822c0..5ef5bd0aea 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1769,7 +1769,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { int64_t itemSize = 0; if ((itemSize = taosQueueItemSize(pInfo->queue)) || RSMA_INFO_ITEM(pInfo, 0)->fetchLevel || RSMA_INFO_ITEM(pInfo, 1)->fetchLevel) { - smaDebug("vgId:%d queueItemSize is %" PRIi64 " execType:%" PRIi8, SMA_VID(pSma), itemSize, type); + smaDebug("vgId:%d, queueItemSize is %" PRIi64 " execType:%" PRIi8, SMA_VID(pSma), itemSize, type); if (atomic_val_compare_exchange_8(&pInfo->assigned, 0, 1) == 0) { taosReadAllQitems(pInfo->queue, pInfo->qall); // queue has mutex lock int32_t qallItemSize = taosQallItemSize(pInfo->qall); @@ -1828,7 +1828,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { } tsem_wait(&pRSmaStat->notEmpty); if (pVnode->inClose && (atomic_load_64(&pRSmaStat->nBufItems) <= 0)) { - smaInfo("prop:vgId:%d loop end check - break - inClose:%d, nBufItems:%" PRIi64, SMA_VID(pSma), pVnode->inClose, + smaInfo("vgId:%d, exec task end, inClose:%d, nBufItems:%" PRIi64, SMA_VID(pSma), pVnode->inClose, atomic_load_64(&pRSmaStat->nBufItems)); break; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 64f223b974..8c73499229 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -220,13 +220,6 @@ int vnodeCommit(SVnode *pVnode) { vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64, TD_VID(pVnode), pVnode->state.commitID, pVnode->state.applied); - // preCommit - // smaSyncPreCommit(pVnode->pSma); - smaAsyncPreCommit(pVnode->pSma); - - vnodeBufPoolUnRef(pVnode->inUse); - pVnode->inUse = NULL; - pVnode->state.commitTerm = pVnode->state.applyTerm; // save info @@ -241,6 +234,16 @@ int vnodeCommit(SVnode *pVnode) { } walBeginSnapshot(pVnode->pWal, pVnode->state.applied); + // preCommit + // smaSyncPreCommit(pVnode->pSma); + if(smaAsyncPreCommit(pVnode->pSma) < 0){ + ASSERT(0); + return -1; + } + + vnodeBufPoolUnRef(pVnode->inUse); + pVnode->inUse = NULL; + // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { ASSERT(0); @@ -248,7 +251,10 @@ int vnodeCommit(SVnode *pVnode) { } if (VND_IS_RSMA(pVnode)) { - smaAsyncCommit(pVnode->pSma); + if (smaAsyncCommit(pVnode->pSma) < 0) { + ASSERT(0); + return -1; + } if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { ASSERT(0); @@ -285,7 +291,10 @@ int vnodeCommit(SVnode *pVnode) { // postCommit // smaSyncPostCommit(pVnode->pSma); - smaAsyncPostCommit(pVnode->pSma); + if (smaAsyncPostCommit(pVnode->pSma) < 0) { + ASSERT(0); + return -1; + } // apply the commit (TODO) walEndSnapshot(pVnode->pWal); From 0de7f3e3a7b29ddd2f2fc08a74481b9074eb3ca5 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 20 Aug 2022 23:46:12 +0800 Subject: [PATCH 172/194] enh: rsma code optimization --- source/dnode/vnode/src/inc/sma.h | 1 - source/dnode/vnode/src/sma/smaRollup.c | 39 +++++++++++--------------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index d32d67e29c..5ba22f6db8 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -123,7 +123,6 @@ struct SRSmaInfoItem { uint8_t nSkipped; // number of skipped to fetch data from all active window int8_t fetchLevel; int32_t maxDelay; // ms - int64_t lastFetch; // ms tmr_h tmrId; }; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 5ef5bd0aea..e2d7a588b0 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -19,9 +19,8 @@ #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid #define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt #define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt -#define RSMA_FETCH_DELAY_MAX (180000) // ms -#define RSMA_FETCH_SKIP_MAX (10) // cnt -#define RSMA_FETCH_ACTIVE_MAX (180) // ms +#define RSMA_FETCH_DELAY_MAX (900000) // ms +#define RSMA_FETCH_ACTIVE_MAX (1800) // ms SSmaMgmt smaMgmt = { .inited = 0, @@ -1646,26 +1645,20 @@ static int32_t tdRSmaConsumeAndFetch(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubm } int64_t curMs = taosGetTimestampMs(); - // if ((pItem->nSkipped > RSMA_FETCH_SKIP_MAX) || (pItem->nSkipped * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) { - // pItem->nSkipped = 0; - // smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch executed", - // SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay); - // } else { - // if (((curMs - pInfo->lastRecv) < RSMA_FETCH_ACTIVE_MAX)) { - // ++pItem->nSkipped; - // smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch skipped ", - // SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); - // continue; - // } else { - // smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ", - // SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); - // } - // } + if ((pItem->nSkipped * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) { + smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch executed", + SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay); + } else if (((curMs - pInfo->lastRecv) < RSMA_FETCH_ACTIVE_MAX)) { + ++pItem->nSkipped; + smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch skipped ", + SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); + continue; + } else { + smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ", + SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); + } - pItem->lastFetch = curMs; - - // smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ", - // SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); + pItem->nSkipped = 0; if ((terrno = qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) { goto _err; @@ -1680,7 +1673,7 @@ static int32_t tdRSmaConsumeAndFetch(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubm SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay); } else { smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 - " maxDelay:%d, fetch not executed as fetchLevel is %" PRIi8, + " maxDelay:%d, fetch not executed as fetch level is %" PRIi8, SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay, pItem->fetchLevel); } } From 38b30d071ed52c4360d2721c70f49c7bfe5c81ce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 00:02:43 +0800 Subject: [PATCH 173/194] fix(query): fix errors. --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 2 +- source/libs/function/src/builtinsimpl.c | 1 + tests/script/tsim/parser/function.sim | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 6a05ad96a6..ab2b2b617a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -938,7 +938,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 if (pSet) { wSet.diskId = pSet->diskId; wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0, .offset = 0}; + fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fData = *pSet->pDataF; fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; fSma = *pSet->pSmaF; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index bf4a07f8e2..5051dcd65c 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5574,6 +5574,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { if (pCtx->end.key != INT64_MIN) { pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end); pInfo->p = pCtx->end; + numOfElems += 1; } pInfo->win.ekey = pInfo->p.key; diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index a27810edeb..0002a5d095 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -129,6 +129,7 @@ if $data03 != 2 then return -1 endi if $data11 != 2.077099980 then + print expect 2.077099980, actual: $data11 return -1 endi if $data12 != 2.077000022 then From 8b3b6a8962384f18e355988368a7806baa4b9462 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 21 Aug 2022 01:07:21 +0800 Subject: [PATCH 174/194] enh: code optimization and memory leak --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 1 - source/dnode/vnode/src/inc/sma.h | 14 +++++++------- source/dnode/vnode/src/inc/vnodeInt.h | 1 - source/dnode/vnode/src/sma/smaCommit.c | 14 +++++++------- source/dnode/vnode/src/sma/smaEnv.c | 6 +++--- source/dnode/vnode/src/sma/smaOpen.c | 8 ++++++-- source/dnode/vnode/src/sma/smaRollup.c | 4 +++- source/libs/tdb/src/db/tdbBtree.c | 3 +++ 8 files changed, 29 insertions(+), 22 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index a309aaf1ca..1f981cc9e0 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -87,7 +87,6 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { while (pVnode->refCount > 0) taosMsleep(10); dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId); - while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 5ba22f6db8..50beb0054d 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -118,12 +118,12 @@ struct SSmaStat { #define RSMA_FS_LOCK(r) (&(r)->lock) struct SRSmaInfoItem { - int8_t level; - int8_t triggerStat; - uint8_t nSkipped; // number of skipped to fetch data from all active window - int8_t fetchLevel; - int32_t maxDelay; // ms - tmr_h tmrId; + int8_t level : 4; + int8_t fetchLevel : 4; + int8_t triggerStat; + uint16_t nSkipped; // number of skipped to fetch data from all active window + int32_t maxDelay; // ms + tmr_h tmrId; }; struct SRSmaInfo { @@ -131,8 +131,8 @@ struct SRSmaInfo { int64_t suid; int64_t refId; // refId of SRSmaStat int64_t lastRecv; // ms - int8_t delFlag; int8_t assigned; // 0 idle, 1 assgined for exec + int8_t delFlag; int16_t padding; T_REF_DECLARE() SRSmaInfoItem items[TSDB_RETENTION_L2]; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 3361b63fd8..8c0ea84bee 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -328,7 +328,6 @@ struct SVnode { SQHandle* pQuery; }; - #define TD_VID(PVNODE) ((PVNODE)->config.vgId) #define VND_TSDB(vnd) ((vnd)->pTsdb) diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index ac9afdb11f..ca5367f397 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -109,7 +109,7 @@ int32_t smaBegin(SSma *pSma) { /** * @brief pre-commit for rollup sma(sync commit). * 1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED. - * 2) wait all triggered fetch tasks finished + * 2) wait for all triggered fetch tasks to finish * 3) perform persist task for qTaskInfo * * @param pSma @@ -127,14 +127,14 @@ static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma) { // step 1: set rsma stat paused atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); - // step 2: wait all triggered fetch tasks finished + // step 2: wait for all triggered fetch tasks to finish int32_t nLoops = 0; while (1) { if (T_REF_VAL_GET(pStat) == 0) { - smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma fetch tasks are all finished", SMA_VID(pSma)); break; } else { - smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma fetch tasks are not all finished yet", SMA_VID(pSma)); } ++nLoops; if (nLoops > 1000) { @@ -319,14 +319,14 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; ASSERT(pRSmaStat->commitAppliedVer > 0); - // step 2: wait all triggered fetch tasks finished + // step 2: wait for all triggered fetch tasks to finish int32_t nLoops = 0; while (1) { if (T_REF_VAL_GET(pStat) == 0) { - smaDebug("vgId:%d, rsma commit, fetch tasks all finished", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma commit, fetch tasks are all finished", SMA_VID(pSma)); break; } else { - smaDebug("vgId:%d, rsma commit, fetch tasks not all finished yet", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma commit, fetch tasks are not all finished yet", SMA_VID(pSma)); } ++nLoops; if (nLoops > 1000) { diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index d39efc748e..e3b83f9955 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -275,14 +275,14 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { } taosHashCleanup(RSMA_INFO_HASH(pStat)); - // step 3: wait all triggered fetch tasks finished + // step 3: wait for all triggered fetch tasks to finish int32_t nLoops = 0; while (1) { if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) { - smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma fetch tasks are all finished", SMA_VID(pSma)); break; } else { - smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma fetch tasks are not all finished yet", SMA_VID(pSma)); } ++nLoops; if (nLoops > 1000) { diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index b2344bc0ec..e2710b26e3 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -148,9 +148,13 @@ int32_t smaClose(SSma *pSma) { int32_t smaPreClose(SSma *pSma) { if (pSma && VND_IS_RSMA(pSma->pVnode)) { - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); + SSmaEnv *pEnv = NULL; + SRSmaStat *pStat = NULL; + if (!(pEnv = SMA_RSMA_ENV(pSma)) || !(pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv))) { + return 0; + } for (int32_t i = 0; i < RSMA_EXECUTOR_MAX; ++i) { - tsem_post(&(pRSmaStat->notEmpty)); + tsem_post(&(pStat->notEmpty)); } } return 0; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index e2d7a588b0..72f779eaab 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1524,7 +1524,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); if (!pStat) { - smaWarn("rsma fetch task not start since rsma stat already destroyed, rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, + smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, pRSmaInfo->refId); return; } @@ -1557,10 +1557,12 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SMA_VID(pSma), pItem->level, pRSmaInfo->suid); // async process pItem->fetchLevel = pItem->level; +#if 0 SRSmaInfo *qInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaInfo->suid); SRSmaInfoItem *qItem = RSMA_INFO_ITEM(qInfo, pItem->level - 1); ASSERT(qItem->level == pItem->level); ASSERT(qItem->fetchLevel == pItem->fetchLevel); +#endif tsem_post(&(pStat->notEmpty)); smaInfo("vgId:%d, rsma fetch task planned for level:%" PRIi8 " suid:%" PRIi64, SMA_VID(pSma), pItem->level, pRSmaInfo->suid); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 7a44edb12c..559b79fd39 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -782,6 +782,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx pBt); tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); tdbOsFree(pNewCell); + if (TDB_CELLDECODER_FREE_VAL(&cd)) { + tdbFree(cd.pVal); + } } // move to next new page From 77bfd400cbd8f3464de5452491f699d93780f3c2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 21 Aug 2022 01:10:18 +0800 Subject: [PATCH 175/194] enh: code optimization --- source/dnode/vnode/src/inc/sma.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 50beb0054d..b36c8eeac3 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -121,7 +121,7 @@ struct SRSmaInfoItem { int8_t level : 4; int8_t fetchLevel : 4; int8_t triggerStat; - uint16_t nSkipped; // number of skipped to fetch data from all active window + uint16_t nSkipped; int32_t maxDelay; // ms tmr_h tmrId; }; From 4b3d98916855c13d72b73c148ade995600efed26 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 21 Aug 2022 01:19:32 +0800 Subject: [PATCH 176/194] other: naming optimization --- source/dnode/vnode/src/sma/smaRollup.c | 67 +++++++------------------- 1 file changed, 18 insertions(+), 49 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 72f779eaab..959a33d8ce 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -42,9 +42,9 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSiz static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid); static void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); static void tdFreeRSmaSubmitItems(SArray *pItems); -static int32_t tdRSmaConsumeAndFetch(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmitArr); -static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, - int64_t suid); +static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmitArr); +static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, + int64_t suid); static void tdRSmaFetchTrigger(void *param, void *tmrId); static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); @@ -636,8 +636,8 @@ _end: return code; } -static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, - int64_t suid) { +static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, + int64_t suid) { SArray *pResList = taosArrayInit(1, POINTER_BYTES); if (pResList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -815,7 +815,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, } SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); - tdRSmaFetchAndSubmitResult(pSma, qTaskInfo, pItem, pInfo->pTSchema, pInfo->suid); + tdRSmaExecAndSubmitResult(pSma, qTaskInfo, pItem, pInfo->pTSchema, pInfo->suid); return TSDB_CODE_SUCCESS; } @@ -1596,46 +1596,15 @@ static void tdFreeRSmaSubmitItems(SArray *pItems) { } } -static int32_t tdRSmaConsumeAndFetch(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmitArr) { - // step 1: consume submit req -#if 0 - int64_t qMemSize = 0; - if ((qMemSize = taosQueueMemorySize(pInfo->queue) > 0)) { - taosReadAllQitems(pInfo->queue, pInfo->qall); // queue has mutex lock - - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); - atomic_fetch_sub_64(&pRSmaStat->nBufItems, taosQallItemSize(pInfo->qall)); - - taosArrayClear(pSubmitArr); - - while (1) { - void *msg = NULL; - taosGetQitem(pInfo->qall, (void **)&msg); - if (msg) { - if (taosArrayPush(pSubmitArr, &msg) < 0) { - tdFreeRSmaSubmitItems(pSubmitArr); - goto _err; - } - } else { - break; - } - } - - int32_t size = taosArrayGetSize(pSubmitArr); - if (size > 0) { - for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { - if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, RSMA_EXEC_TIMEOUT, i) < - 0) { - tdFreeRSmaSubmitItems(pSubmitArr); - goto _err; - } - } - - tdFreeRSmaSubmitItems(pSubmitArr); - } - } -#endif - // step 2: fetch rsma result(consider the efficiency and functionality) +/** + * @brief fetch rsma result(consider the efficiency and functionality) + * + * @param pSma + * @param pInfo + * @param pSubmitArr + * @return int32_t + */ +static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmitArr) { SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; for (int8_t i = 1; i <= TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, i - 1); @@ -1665,7 +1634,7 @@ static int32_t tdRSmaConsumeAndFetch(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubm if ((terrno = qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) { goto _err; } - if (tdRSmaFetchAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid) < 0) { + if (tdRSmaExecAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid) < 0) { tdCleanupStreamInputDataBlock(taskInfo); goto _err; } @@ -1772,7 +1741,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { tdRSmaBatchExec(pSma, pInfo, pInfo->qall, pSubmitArr, type); } - tdRSmaConsumeAndFetch(pSma, pInfo, pSubmitArr); + tdRSmaFetchAllResult(pSma, pInfo, pSubmitArr); if (qallItemSize > 0) { // subtract the item size after the task finished, commit should wait for all items be consumed atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize); @@ -1803,7 +1772,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { tdRSmaBatchExec(pSma, pInfo, pInfo->qall, pSubmitArr, type); } - // tdRSmaConsumeAndFetch(pSma, pInfo, pSubmitArr); + // tdRSmaFetchAllResult(pSma, pInfo, pSubmitArr); ASSERT(1 == atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0)); } } From 65e1cce6dfbc3b94bc18c757775c5c1dad0192d9 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 21 Aug 2022 01:22:56 +0800 Subject: [PATCH 177/194] enh: rsma code optimization --- source/dnode/vnode/src/sma/smaRollup.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 959a33d8ce..448b8ab508 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -162,8 +162,8 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { pInfo->iQall = NULL; } - taosMemoryFree(pInfo); - } + taosMemoryFree(pInfo); + } return NULL; } @@ -1524,8 +1524,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); if (!pStat) { - smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, - pRSmaInfo->refId); + smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%" PRIi64 " refId:%d)", + smaMgmt.rsetId, pRSmaInfo->refId); return; } @@ -1741,7 +1741,10 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { tdRSmaBatchExec(pSma, pInfo, pInfo->qall, pSubmitArr, type); } - tdRSmaFetchAllResult(pSma, pInfo, pSubmitArr); + if (type == RSMA_EXEC_OVERFLOW) { + tdRSmaFetchAllResult(pSma, pInfo, pSubmitArr); + } + if (qallItemSize > 0) { // subtract the item size after the task finished, commit should wait for all items be consumed atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize); From 5324889c52abe2833f657063194c8c84f696ce8b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 21 Aug 2022 11:26:20 +0800 Subject: [PATCH 178/194] fix: sma file corrupt --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index a8969b2d9c..c8f3862071 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -694,8 +694,18 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; + // seek + int64_t n = taosLSeekFile(pReader->pSmaFD, pSmaInfo->offset, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n < pSmaInfo->offset) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + // read - int64_t n = taosReadFile(pReader->pSmaFD, pReader->aBuf[0], size); + n = taosReadFile(pReader->pSmaFD, pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; From e0317964f096b69ddcea5a1bc19f31165ff5e842 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 21 Aug 2022 14:56:20 +0800 Subject: [PATCH 179/194] fix: adjust max executor for rsma --- source/dnode/vnode/src/inc/sma.h | 2 +- source/libs/tdb/src/db/tdbBtree.c | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index b36c8eeac3..ca77042bb2 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -33,7 +33,7 @@ extern "C" { // clang-format on #define RSMA_TASK_INFO_HASH_SLOT (8) -#define RSMA_EXECUTOR_MAX (4) +#define RSMA_EXECUTOR_MAX (1) typedef struct SSmaEnv SSmaEnv; typedef struct SSmaStat SSmaStat; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 559b79fd39..7a44edb12c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -782,9 +782,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx pBt); tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); tdbOsFree(pNewCell); - if (TDB_CELLDECODER_FREE_VAL(&cd)) { - tdbFree(cd.pVal); - } } // move to next new page From 470c774f54fb03a842c8752ea32f334a51e7bd91 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 21 Aug 2022 15:05:56 +0800 Subject: [PATCH 180/194] fix: alter table schema --- source/dnode/vnode/src/meta/metaTable.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index c1e59adbb0..aa107ab253 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -357,10 +357,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { metaSaveToTbDb(pMeta, &nStbEntry); // update uid index - SMetaInfo info; - metaGetEntryInfo(&nStbEntry, &info); - tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), - &(SUidIdxVal){.suid = info.suid, .version = info.version, .skmVer = info.skmVer}, sizeof(SUidIdxVal), 0); + metaUpdateUidIdx(pMeta, &nStbEntry); if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); metaULock(pMeta); @@ -884,7 +881,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA } SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; - tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag*)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); + tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, + ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); tDecoderClear(&dc1); tDecoderClear(&dc2); @@ -1091,7 +1089,8 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, ((STag*)(pME->ctbEntry.pTags))->len, &pMeta->txn); + return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, + ((STag *)(pME->ctbEntry.pTags))->len, &pMeta->txn); } int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, From fe918b3f69443f2af549c404c22c5b0d5b0356e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 15:56:06 +0800 Subject: [PATCH 181/194] fix(query): set correct column data. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c7e6d381a2..0e2a4e47bf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -814,19 +814,20 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn pColData = taosArrayGet(pResBlock->pDataBlock, i); SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex); - - if (pData->cid == pColData->info.colId) { + if (pData->cid < pColData->info.colId) { + colIndex += 1; + } else if (pData->cid == pColData->info.colId) { for (int32_t j = pDumpInfo->rowIndex; j < endIndex && j >= 0; j += step) { tColDataGetValue(pData, j, &cv); doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); } colIndex += 1; + i += 1; ASSERT(rowIndex == remain); } else { // the specified column does not exist in file block, fill with null data colDataAppendNNULL(pColData, 0, remain); + i += 1; } - - i += 1; } while (i < numOfOutputCols) { @@ -2436,8 +2437,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { if (pLastBlockReader->currentBlockIndex != -1) { initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || - pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { + int32_t index = pScanInfo->indexInBlockL; + if (index == DEFAULT_ROW_INDEX_VAL || index == pLastBlockReader->lastBlockData.nRow) { bool hasData = nextRowInLastBlock(pLastBlockReader); if (!hasData) { // current table does not have rows in last block, try next table pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); From d305e975c44a7982a2d70e92b88dea04a3d44914 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 15:56:46 +0800 Subject: [PATCH 182/194] fix(query): remove printf. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0e2a4e47bf..d89b45b140 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2506,7 +2506,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (pLastBlockReader->currentBlockIndex == -1) { // ASSERT(0); - printf("error\n"); } initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); From c39152ed869622f27e9d47816ab516e706c5ff98 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 21 Aug 2022 18:30:06 +0800 Subject: [PATCH 183/194] Update index.md --- docs/en/02-intro/index.md | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/docs/en/02-intro/index.md b/docs/en/02-intro/index.md index f6766f910f..f2bc6fd528 100644 --- a/docs/en/02-intro/index.md +++ b/docs/en/02-intro/index.md @@ -3,7 +3,7 @@ title: Introduction toc_max_heading_level: 2 --- -TDengine is a high-performance, scalable time-series database with SQL support. Its code, including its cluster feature is open source under GNU AGPL v3.0. Besides the database engine, it provides [caching](/develop/cache), [stream processing](/develop/continuous-query), [data subscription](/develop/subscribe) and other functionalities to reduce the complexity and cost of development and operation. +TDengine is an open source, high-performance, cloud native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. Its code, including its cluster feature is open source under GNU AGPL v3.0. Besides the database engine, it provides [caching](/develop/cache), [stream processing](/develop/continuous-query), [data subscription](/develop/subscribe) and other functionalities to reduce the system complexity and cost of development and operation. This section introduces the major features, competitive advantages, typical use-cases and benchmarks to help you get a high level overview of TDengine. @@ -31,25 +31,21 @@ For more details on features, please read through the entire documentation. ## Competitive Advantages -Time-series data is structured, not transactional, and is rarely deleted or updated. TDengine makes full use of [these characteristics of time series data](https://tdengine.com/2019/07/09/86.html) to build its own innovative storage engine and computing engine to differentiate itself from other time series databases, with the following advantages. +By making full use of [these characteristics of time series data](https://tdengine.com/2019/07/09/86.html), TDengine differentiates itself from other time series databases, with the following advantages. -- **[High Performance](https://tdengine.com/fast)**: With an innovatively designed and purpose-built storage engine, TDengine outperforms other time series databases in data ingestion and querying while significantly reducing storage costs and compute costs. +- **High-Performance**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression. -- **[Scalable](https://tdengine.com/scalable)**: TDengine provides out-of-box scalability and high-availability through its native distributed design. Nodes can be added through simple configuration to achieve greater data processing power. In addition, this feature is open source. +- **Simplified Solution**: Through built-in caching, stream processing and data subscription features, TDengine provides a simplified solution for time-series data processing. It reduces system design complexity and operation costs significantly. -- **[SQL Support](https://tdengine.com/sql-support)**: TDengine uses SQL as the query language, thereby reducing learning and migration costs, while adding SQL extensions to better handle time-series. Keeping NoSQL developers in mind, TDengine also supports convenient and flexible, schemaless data ingestion. +- **Cloud Native**: Through native distributed design, sharding and partitioning, separation of compute and storage, RAFT, support for kubernetes deployment and full observability, TDengine is a cloud native Time-Series Database and can be deployed on public, private or hybrid clouds. -- **All in One**: TDengine has built-in caching, stream processing and data subscription functions. It is no longer necessary to integrate Kafka/Redis/HBase/Spark or other software in some scenarios. It makes the system architecture much simpler, cost-effective and easier to maintain. +- **Ease of Use**: For administrators, TDengine significantly reduces the effort to deploy and maintain. For developers, it provides a simple interface, simplified solution and seamless integrations for third party tools. For data users, it gives easy data access. -- **Seamless Integration**: Without a single line of code, TDengine provide seamless, configurable integration with third-party tools such as Telegraf, Grafana, EMQX, Prometheus, StatsD, collectd, etc. More third-party tools are being integrated. +- **Easy Data Analytics**: Through super tables, storage and compute separation, data partitioning by time interval, pre-computation and other means, TDengine makes it easy to explore, format, and get access to data in a highly efficient way. -- **Zero Management**: Installation and cluster setup can be done in seconds. Data partitioning and sharding are executed automatically. TDengine’s running status can be monitored via Grafana or other DevOps tools. +- **Open Source**: TDengine’s core modules, including cluster feature, are all available under open source licenses. It has gathered 18.8k stars on GitHub. There is an active developer community, and over 139k running instances worldwide. -- **Zero Learning Costs**: With SQL as the query language and support for ubiquitous tools like Python, Java, C/C++, Go, Rust, and Node.js connectors, and a REST API, there are zero learning costs. - -- **Interactive Console**: TDengine provides convenient console access to the database, through a CLI, to run ad hoc queries, maintain the database, or manage the cluster, without any programming. - -With TDengine, the total cost of ownership of your time-series data platform can be greatly reduced. 1: With its superior performance, the computing and storage resources are reduced significantly 2: With SQL support, it can be seamlessly integrated with many third party tools, and learning costs/migration costs are reduced significantly 3: With its simple architecture and zero management, the operation and maintenance costs are reduced. +With TDengine, the total cost of ownership of your time-series data platform can be greatly reduced. 1: With its superior performance, the computing and storage resources are reduced significantly 2: With SQL support, it can be seamlessly integrated with many third party tools, and learning costs/migration costs are reduced significantly 3: With its simplified solution and nearly zero management, the operation and maintenance costs are reduced significantly. ## Technical Ecosystem This is how TDengine would be situated, in a typical time-series data processing platform: From 319a2c040fa646d0e62e2b0d2cd9fa179ee8a9e3 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 21 Aug 2022 18:35:38 +0800 Subject: [PATCH 184/194] Update 01-index.md --- docs/en/01-index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/en/01-index.md b/docs/en/01-index.md index f5b7f3e0f6..0a09f860c4 100644 --- a/docs/en/01-index.md +++ b/docs/en/01-index.md @@ -4,7 +4,8 @@ sidebar_label: Documentation Home slug: / --- -TDengine is a [high-performance](https://tdengine.com/fast), [scalable](https://tdengine.com/scalable) time series database with [SQL support](https://tdengine.com/sql-support). This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design and other topics. It’s written mainly for architects, developers and system administrators. + +TDengine is an open source, cloud native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. . This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design and other topics. It’s written mainly for architects, developers and system administrators. To get an overview of TDengine, such as a feature list, benchmarks, and competitive advantages, please browse through the [Introduction](./intro) section. From e68788b646c75e5beaa8cc3690c283aeef4966bf Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 21 Aug 2022 18:36:09 +0800 Subject: [PATCH 185/194] Update 01-index.md --- docs/en/01-index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/01-index.md b/docs/en/01-index.md index 0a09f860c4..4c2f9b02b9 100644 --- a/docs/en/01-index.md +++ b/docs/en/01-index.md @@ -5,7 +5,7 @@ slug: / --- -TDengine is an open source, cloud native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. . This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design and other topics. It’s written mainly for architects, developers and system administrators. +TDengine is an open source, cloud native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design and other topics. It’s written mainly for architects, developers and system administrators. To get an overview of TDengine, such as a feature list, benchmarks, and competitive advantages, please browse through the [Introduction](./intro) section. From 8b4b9f39676dec373a30893842353ee00e791df0 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 21 Aug 2022 18:58:28 +0800 Subject: [PATCH 186/194] Update index.mdx --- docs/en/07-develop/02-model/index.mdx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/07-develop/02-model/index.mdx b/docs/en/07-develop/02-model/index.mdx index 06a5346141..e200920a76 100644 --- a/docs/en/07-develop/02-model/index.mdx +++ b/docs/en/07-develop/02-model/index.mdx @@ -2,7 +2,9 @@ title: Data Model --- -The data model employed by TDengine is similar to that of a relational database. You have to create databases and tables. You must design the data model based on your own business and application requirements. You should design the STable (an abbreviation for super table) schema to fit your data. This chapter will explain the big picture without getting into syntactical details. +The data model employed by TDengine is similar to that of a relational database. You have to create databases and tables. You must design the data model based on your own business and application requirements. You should design the [STable](/concept/#super-table-stable) (an abbreviation for super table) schema to fit your data. This chapter will explain the big picture without getting into syntactical details. + +Note: before you read this chapter, please make sure you have already read through [Key Concepts](/concept/), since TDengine introduces new concepts like "one table for one [data collection point](/concept/#data-collection-point)" and "[super table](/concept/#super-table-stable)". ## Create Database From 7904686915e34967973e87e2d6e09c63d7cb6427 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 18:59:21 +0800 Subject: [PATCH 187/194] fix(query): support delete in last --- source/dnode/vnode/src/tsdb/tsdbRead.c | 54 +++++++++++++------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d89b45b140..6c9042f64f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -166,7 +166,7 @@ static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i static TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader, SRowMerger* pMerger); -static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, SRowMerger* pMerger); +static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); @@ -1441,7 +1441,7 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf init = true; tRowMergerInit(&merge, &fRow1, pReader->pSchema); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } if (minKey == k.ts) { @@ -1470,7 +1470,7 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf init = true; tRowMergerInit(&merge, &fRow1, pReader->pSchema); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } if (minKey == key) { @@ -1641,7 +1641,7 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo init = true; tRowMergerInit(&merge, &fRow1, pReader->pSchema); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } if (minKey == ik.ts) { @@ -1691,7 +1691,7 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo init = true; tRowMergerInit(&merge, &fRow1, pReader->pSchema); } - doMergeRowsInLastBlock(pLastBlockReader, tsLast, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } if (minKey == key) { @@ -1904,7 +1904,7 @@ static void setAllRowsChecked(SLastBlockReader *pLastBlockReader) { *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; } -static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { +static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { int32_t step = (pLastBlockReader->order == TSDB_ORDER_ASC) ? 1 : -1; if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { return false; @@ -1918,21 +1918,28 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader) { continue; } - if (pBlockData->aTSKEY[i] < pLastBlockReader->window.skey) { + int64_t ts = pBlockData->aTSKEY[i]; + if (ts < pLastBlockReader->window.skey) { continue; } - if (pBlockData->aVersion[i] < pLastBlockReader->verRange.minVer) { + int64_t ver = pBlockData->aVersion[i]; + if (ver < pLastBlockReader->verRange.minVer) { continue; } // no data any more, todo opt handle desc case - if (pBlockData->aTSKEY[i] > pLastBlockReader->window.ekey) { + if (ts > pLastBlockReader->window.ekey) { continue; } // todo opt handle desc case - if (pBlockData->aVersion[i] > pLastBlockReader->verRange.maxVer) { + if (ver > pLastBlockReader->verRange.maxVer) { + continue; + } + + TSDBKEY k = {.ts = ts, .version = ver}; + if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k, pLastBlockReader->order)) { continue; } @@ -1958,7 +1965,8 @@ static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { } // todo refactor -static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader *pLastBlockReader) { +static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, + SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t key = (pBlockData->nRow > 0)? pBlockData->aTSKEY[pDumpInfo->rowIndex]:INT64_MIN; @@ -2011,7 +2019,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow1, pReader->pSchema); - doMergeRowsInLastBlock(pLastBlockReader, ts, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); @@ -2025,7 +2033,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - doMergeRowsInLastBlock(pLastBlockReader, ts, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); @@ -2061,7 +2069,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInLastBlock(pLastBlockReader, tsLastBlock, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); @@ -2085,10 +2093,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pBlockScanInfo = pReader->status.pTableIter; } - SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; -// initLastBlockReader(pLastBlockReader, pBlockScanInfo->uid, &pBlockScanInfo->indexInBlockL); -// bool has = nextRowInLastBlock(pLastBlockReader); // todo handle if no data in last block - + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; @@ -2439,7 +2444,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); int32_t index = pScanInfo->indexInBlockL; if (index == DEFAULT_ROW_INDEX_VAL || index == pLastBlockReader->lastBlockData.nRow) { - bool hasData = nextRowInLastBlock(pLastBlockReader); + bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); if (!hasData) { // current table does not have rows in last block, try next table pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); if (pStatus->pTableIter == NULL) { @@ -2504,13 +2509,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } - if (pLastBlockReader->currentBlockIndex == -1) { -// ASSERT(0); - } - + // note: the lastblock may be null here initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { - bool hasData = nextRowInLastBlock(pLastBlockReader); + bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); } } @@ -3034,8 +3036,8 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc } // todo check if the rows are dropped or not -int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, int64_t ts, SRowMerger* pMerger) { - while(nextRowInLastBlock(pLastBlockReader)) { +int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger) { + while(nextRowInLastBlock(pLastBlockReader, pScanInfo)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, *pLastBlockReader->rowIndex); From 9a85b8011a53b76aac87de00097471e40d7e5abe Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 21 Aug 2022 19:12:34 +0800 Subject: [PATCH 188/194] Update index.md --- docs/en/02-intro/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/02-intro/index.md b/docs/en/02-intro/index.md index f2bc6fd528..23a79aa229 100644 --- a/docs/en/02-intro/index.md +++ b/docs/en/02-intro/index.md @@ -31,7 +31,7 @@ For more details on features, please read through the entire documentation. ## Competitive Advantages -By making full use of [these characteristics of time series data](https://tdengine.com/2019/07/09/86.html), TDengine differentiates itself from other time series databases, with the following advantages. +By making full use of [characteristics of time series data](https://tdengine.com/tsdb/characteristics-of-time-series-data/), TDengine differentiates itself from other time series databases, with the following advantages. - **High-Performance**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression. @@ -45,7 +45,7 @@ By making full use of [these characteristics of time series data](https://tdengi - **Open Source**: TDengine’s core modules, including cluster feature, are all available under open source licenses. It has gathered 18.8k stars on GitHub. There is an active developer community, and over 139k running instances worldwide. -With TDengine, the total cost of ownership of your time-series data platform can be greatly reduced. 1: With its superior performance, the computing and storage resources are reduced significantly 2: With SQL support, it can be seamlessly integrated with many third party tools, and learning costs/migration costs are reduced significantly 3: With its simplified solution and nearly zero management, the operation and maintenance costs are reduced significantly. +With TDengine, the total cost of ownership of your time-series data platform can be greatly reduced. 1: With its superior performance, the computing and storage resources are reduced significantly；2: With SQL support, it can be seamlessly integrated with many third party tools, and learning costs/migration costs are reduced significantly；3: With its simplified solution and nearly zero management, the operation and maintenance costs are reduced significantly. ## Technical Ecosystem This is how TDengine would be situated, in a typical time-series data processing platform: From 774c86d44ab9eadae8a1806c0357494227de2a3f Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 21 Aug 2022 19:14:38 +0800 Subject: [PATCH 189/194] Update index.md --- docs/en/04-concept/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/04-concept/index.md b/docs/en/04-concept/index.md index 850f705146..e986c146b1 100644 --- a/docs/en/04-concept/index.md +++ b/docs/en/04-concept/index.md @@ -2,7 +2,7 @@ title: Concepts --- -In order to explain the basic concepts and provide some sample code, the TDengine documentation smart meters as a typical time series use case. We assume the following: 1. Each smart meter collects three metrics i.e. current, voltage, and phase 2. There are multiple smart meters, and 3. Each meter has static attributes like location and group ID. Based on this, collected data will look similar to the following table: +In order to explain the basic concepts and provide some sample code, the TDengine documentation smart meters as a typical time series use case. We assume the following: 1. Each smart meter collects three metrics i.e. current, voltage, and phase; 2. There are multiple smart meters; 3. Each meter has static attributes like location and group ID. Based on this, collected data will look similar to the following table:
From 29a1245be4d2ac830481e0c5d61b73fb5e00bdf2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 20:15:53 +0800 Subject: [PATCH 190/194] fix(query): use the separate last block delete index. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6c9042f64f..fc7595627d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -45,7 +45,8 @@ typedef struct STableBlockScanInfo { SIterInfo iter; // mem buffer skip list iterator SIterInfo iiter; // imem buffer skip list iterator SArray* delSkyline; // delete info for this table - int32_t fileDelIndex; + int32_t fileDelIndex; // file block delete index + int32_t lastBlockDelIndex;// delete index for last block bool iterInit; // whether to initialize the in-memory skip list iterator or not int16_t indexInBlockL;// row position in last block } STableBlockScanInfo; @@ -1296,7 +1297,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc // todo here we need to each key in the last files to identify if it is really overlapped with last block bool overlapWithlastBlock = false; - if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0) { + if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0 && (pLastBlockReader->currentBlockIndex != -1)) { SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); } @@ -1939,7 +1940,7 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader, STableBlockSc } TSDBKEY k = {.ts = ts, .version = ver}; - if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k, pLastBlockReader->order)) { + if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) { continue; } @@ -2295,6 +2296,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* ASCENDING_TRAVERSE(pReader->order) ? 0 : taosArrayGetSize(pBlockScanInfo->delSkyline) - 1; pBlockScanInfo->iiter.index = pBlockScanInfo->iter.index; pBlockScanInfo->fileDelIndex = pBlockScanInfo->iter.index; + pBlockScanInfo->lastBlockDelIndex = pBlockScanInfo->iter.index; return code; _err: @@ -2305,9 +2307,6 @@ _err: static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; -// SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); -// STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); - initMemDataIterator(pScanInfo, pReader); TSDBROW* pRow = getValidRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); if (pRow != NULL) { @@ -2381,10 +2380,12 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return TSDB_CODE_SUCCESS; } -static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, uint64_t uid, STsdbReader* pReader) { +static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo *pBlockScanInfo, STsdbReader* pReader) { SArray* pBlocks = pLastBlockReader->pBlockL; SBlockL* pBlock = NULL; + uint64_t uid = pBlockScanInfo->uid; + initMemDataIterator(pBlockScanInfo, pReader); pLastBlockReader->currentBlockIndex = -1; // find the correct SBlockL @@ -2435,7 +2436,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // load the last data block of current table // todo opt perf by avoiding load last block repeatly STableBlockScanInfo* pScanInfo = pStatus->pTableIter; - int32_t code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); + int32_t code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2504,7 +2505,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { key = getCurrentKeyInBuf(pScanInfo, pReader); // load the last data block of current table - code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo->uid, pReader); + code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } From 4879ae3d68e904d0ba0671ba74fbd856f9e10a22 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 21:36:15 +0800 Subject: [PATCH 191/194] refactor(query): optimize load last block. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 37 +++++++++++++++++--------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index fc7595627d..64a992fbdc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -94,7 +94,7 @@ typedef struct SLastBlockReader { SVersionRange verRange; int32_t order; uint64_t uid; - int16_t* rowIndex; // row index ptr, usually from the STableBlockScanInfo->indexInBlockL + int16_t* rowIndex; // row index ptr, usually from the STableBlockScanInfo->indexInBlockL } SLastBlockReader; typedef struct SFilesetIter { @@ -2380,44 +2380,57 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return TSDB_CODE_SUCCESS; } +// todo add elapsed time results static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo *pBlockScanInfo, STsdbReader* pReader) { SArray* pBlocks = pLastBlockReader->pBlockL; SBlockL* pBlock = NULL; uint64_t uid = pBlockScanInfo->uid; - initMemDataIterator(pBlockScanInfo, pReader); - pLastBlockReader->currentBlockIndex = -1; + int32_t totalLastBlocks = (int32_t)taosArrayGetSize(pBlocks); - // find the correct SBlockL - for (int32_t i = 0; i < taosArrayGetSize(pBlocks); ++i) { + initMemDataIterator(pBlockScanInfo, pReader); + + // find the correct SBlockL. todo binary search + int32_t index = -1; + for (int32_t i = 0; i < totalLastBlocks; ++i) { SBlockL* p = taosArrayGet(pBlocks, i); if (p->minUid <= uid && p->maxUid >= uid) { - pLastBlockReader->currentBlockIndex = i; + index = i; pBlock = p; break; } } - if (pLastBlockReader->currentBlockIndex == -1) { + if (index == -1) { + pLastBlockReader->currentBlockIndex = index; tBlockDataReset(&pLastBlockReader->lastBlockData); return TSDB_CODE_SUCCESS; } + // the required last datablock has already loaded + if (index == pLastBlockReader->currentBlockIndex) { + return TSDB_CODE_SUCCESS; + } + int32_t code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p init block data failed, code:%s %s", pReader, tstrerror(code), pReader->idStr); return code; } + ; code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData); if (code != TSDB_CODE_SUCCESS) { - tsdbError( - "%p error occurs in loading last block into buffer, last block index:%d, total:%d rows:%d, minVer:%" PRId64 - ", maxVer:%" PRId64 ", code:%s %s", - pReader, pLastBlockReader->currentBlockIndex, (int32_t)taosArrayGetSize(pBlocks), pBlock->nRow, pBlock->minVer, - pBlock->maxVer, tstrerror(code), pReader->idStr); + tsdbError("%p error occurs in loading last block into buffer, last block index:%d, total:%d code:%s %s", pReader, + pLastBlockReader->currentBlockIndex, totalLastBlocks, tstrerror(code), pReader->idStr); + } else { + tsdbDebug("%p load last block completed, uid:%" PRIu64 + " last block index:%d, total:%d rows:%d, minVer:%d, maxVer:%d, brange:%" PRId64 " - %" PRId64 " %s", + pReader, uid, pLastBlockReader->currentBlockIndex, totalLastBlocks, pBlock->nRow, pBlock->minVer, + pBlock->maxVer, pBlock->minKey, pBlock->maxKey, pReader->idStr); } + pLastBlockReader->currentBlockIndex = index; return TSDB_CODE_SUCCESS; } From a7ba94939a1a2ab117b461dcdef77801b3a6d18e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 22:02:01 +0800 Subject: [PATCH 192/194] fix(query): set init value for current last block index. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 64a992fbdc..76a7484fb9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -339,6 +339,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb pLReader->order = pReader->order; pLReader->window = pReader->window; pLReader->verRange = pReader->verRange; + pLReader->currentBlockIndex = -1; int32_t code = tBlockDataCreate(&pLReader->lastBlockData); if (code != TSDB_CODE_SUCCESS) { @@ -2418,7 +2419,6 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STable return code; } - ; code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p error occurs in loading last block into buffer, last block index:%d, total:%d code:%s %s", pReader, From f029df4876da8df8913deb35e678cf7b633b4e1f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Aug 2022 22:17:12 +0800 Subject: [PATCH 193/194] fix(query): reset index when moving to next file. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 76a7484fb9..336053911e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2327,15 +2327,17 @@ static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* p static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { SReaderStatus* pStatus = &pReader->status; + pBlockNum->numOfBlocks = 0; + pBlockNum->numOfLastBlocks = 0; size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); SArray* pLastBlocks = pStatus->fileIter.pLastBlockReader->pBlockL; + taosArrayClear(pLastBlocks); while (1) { bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader); if (!hasNext) { // no data files on disk - taosArrayClear(pLastBlocks); break; } @@ -2631,6 +2633,9 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl resetDataBlockIterator(pBlockIter, pReader->order, pReader->status.pTableMap); } + SLastBlockReader* pLReader = pReader->status.fileIter.pLastBlockReader; + pLReader->currentBlockIndex = -1; + // set the correct start position according to the query time window initBlockDumpInfo(pReader, pBlockIter); return code; From e5ce3282ba968aab451c7887e3a6b8b48b9a76b2 Mon Sep 17 00:00:00 2001 From: Raistlin Chan Date: Sun, 21 Aug 2022 23:43:22 +0800 Subject: [PATCH 194/194] Update 03-table.md (#16236) fix 'ODIFY' -> 'MODIFY' --- docs/zh/12-taos-sql/03-table.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 1e20f73541..0e104bb7b6 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -110,7 +110,7 @@ alter_table_option: { 对普通表可以进行如下修改操作 1. ADD COLUMN：添加列。 2. DROP COLUMN：删除列。 -3. ODIFY COLUMN：修改列定义，如果数据列的类型是可变长类型，那么可以使用此指令修改其宽度，只能改大，不能改小。 +3. MODIFY COLUMN：修改列定义，如果数据列的类型是可变长类型，那么可以使用此指令修改其宽度，只能改大，不能改小。 4. RENAME COLUMN：修改列名称。 ### 增加列 @@ -195,4 +195,4 @@ SHOW CREATE TABLE tb_name; ``` DESCRIBE [db_name.]tb_name; -``` \ No newline at end of file +```