From 726006d18a869ff05eafc91162fc24381e88a217 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 22 Aug 2022 17:37:51 +0800 Subject: [PATCH 001/102] more code --- source/dnode/vnode/CMakeLists.txt | 2 + source/dnode/vnode/src/tsdb/tsdbCompress.c | 64 +++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbDiskData.c | 84 ++++++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 source/dnode/vnode/src/tsdb/tsdbCompress.c create mode 100644 source/dnode/vnode/src/tsdb/tsdbDiskData.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a3e17f5377..0944a6efbc 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -49,6 +49,8 @@ target_sources( "src/tsdb/tsdbSnapshot.c" "src/tsdb/tsdbCacheRead.c" "src/tsdb/tsdbRetention.c" + "src/tsdb/tsdbDiskData.c" + "src/tsdb/tsdbCompress.c" # tq "src/tq/tq.c" diff --git a/source/dnode/vnode/src/tsdb/tsdbCompress.c b/source/dnode/vnode/src/tsdb/tsdbCompress.c new file mode 100644 index 0000000000..76be7c1070 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbCompress.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +// Integer ===================================================== +typedef struct { + int8_t rawCopy; + int64_t prevVal; + int32_t nVal; + int32_t nBuf; + uint8_t *pBuf; +} SIntCompressor; + +#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) +#define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL) + +static int32_t tsdbCmprI64(SIntCompressor *pCompressor, int64_t val) { + int32_t code = 0; + + // raw copy + if (pCompressor->rawCopy) { + memcpy(pCompressor->pBuf + pCompressor->nBuf, &val, sizeof(val)); + pCompressor->nBuf += sizeof(val); + pCompressor->nVal++; + goto _exit; + } + + if (!I64_SAFE_ADD(val, pCompressor->prevVal)) { + pCompressor->rawCopy = 1; + // TODO: decompress and copy + pCompressor->nVal++; + goto _exit; + } + + int64_t diff = val - pCompressor->prevVal; + uint8_t zigzag = ZIGZAGE(int64_t, diff); + + if (zigzag >= SIMPLE8B_MAX) { + pCompressor->rawCopy = 1; + // TODO: decompress and copy + pCompressor->nVal++; + goto _exit; + } + +_exit: + return code; +} + +// Timestamp ===================================================== + +// Float ===================================================== \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c new file mode 100644 index 0000000000..3bd71f0ea6 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +typedef struct SDiskColBuilder SDiskColBuilder; +struct SDiskColBuilder { + uint8_t flags; + uint8_t *pBitMap; + int32_t *aOffset; + int32_t nData; + uint8_t *pData; +}; + +int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + // TODO + return code; +} + +// ================================================================ +typedef struct SDiskDataBuilder SDiskDataBuilder; +struct SDiskDataBuilder { + SDiskDataHdr hdr; + SArray *aBlockCol; // SArray +}; + +int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { + int32_t code = 0; + // TODO + return code; +} + +void tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { + // TODO +} + +void tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, int64_t suid, int64_t uid, STSchema *pTSchema, int8_t cmprAlg) { + pBuilder->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, // + .fmtVer = 0, + .suid = suid, + .uid = uid, + .cmprAlg = cmprAlg}; +} + +void tDiskDataBuilderReset(SDiskDataBuilder *pBuilder) { + // TODO +} + +int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) { + int32_t code = 0; + + // uid (todo) + + // version (todo) + + // TSKEY (todo) + + SRowIter iter = {0}; + tRowIterInit(&iter, pRow, pTSchema); + + for (int32_t iDiskCol = 0; iDiskCol < 0; iDiskCol++) { + } + + return code; +} + +int32_t tDiskDataBuilderGet(SDiskDataBuilder *pBuilder, uint8_t **ppData) { + int32_t code = 0; + // TODO + return code; +} \ No newline at end of file From 3f90f99d8f0f8724df5e5771baec98828be77f6e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 22 Aug 2022 18:12:40 +0800 Subject: [PATCH 002/102] first step optimization --- source/dnode/vnode/src/inc/tsdb.h | 4 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 115 +++++++----------- source/dnode/vnode/src/tsdb/tsdbFile.c | 39 +++++- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 30 ++--- source/dnode/vnode/src/tsdb/tsdbRetention.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- 7 files changed, 104 insertions(+), 94 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d1f5cfb122..0cac1faa25 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -67,6 +67,7 @@ typedef struct SBlockCol SBlockCol; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 +#define TSDB_MAX_LAST_FILE 16 #define TSDB_FHDR_SIZE 512 #define HAS_NONE ((int8_t)0x1) @@ -556,8 +557,9 @@ struct SDFileSet { int32_t fid; SHeadFile *pHeadF; SDataFile *pDataF; - SLastFile *pLastF; SSmaFile *pSmaF; + uint8_t nLastF; + SLastFile *aLastF[TSDB_MAX_LAST_FILE]; }; struct SRowIter { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 020f3b0bc6..59afec00ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -458,10 +458,11 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { SDataFile fData; SLastFile fLast; SSmaFile fSma; - SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma}; + SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aLastF[0] = &fLast, .pSmaF = &fSma}; if (pRSet) { wSet.diskId = pRSet->diskId; wSet.fid = pCommitter->commitFid; + wSet.nLastF = 1; fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; fData = *pRSet->pDataF; fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; @@ -475,6 +476,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { wSet.diskId = did; wSet.fid = pCommitter->commitFid; + wSet.nLastF = 1; fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; fData = (SDataFile){.commitID = pCommitter->commitID, .size = 0}; fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 247de99338..d86cb7ab9b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -254,8 +254,10 @@ void tsdbFSDestroy(STsdbFS *pFS) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet); taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->pLastF); taosMemoryFree(pSet->pSmaF); + for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { + taosMemoryFree(pSet->aLastF[iLast]); + } } taosArrayDestroy(pFS->aDFileSet); @@ -310,12 +312,12 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { } // last =========== - tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname); + tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[0], fname); if (taosStatFile(fname, &size, NULL)) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size != pSet->pLastF->size) { + if (size != pSet->aLastF[0]->size) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -382,41 +384,15 @@ static int32_t tsdbRecoverFS(STsdb *pTsdb, uint8_t *pData, int64_t nData) { taosArrayClear(pTsdb->fs.aDFileSet); n += tGetU32v(pData + n, &nSet); for (uint32_t iSet = 0; iSet < nSet; iSet++) { - SDFileSet fSet; + SDFileSet fSet = {0}; - // head - fSet.pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile)); - if (fSet.pHeadF == NULL) { + int32_t nt = tGetDFileSet(pData + n, &fSet); + if (nt < 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - fSet.pHeadF->nRef = 1; - // data - fSet.pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile)); - if (fSet.pDataF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - fSet.pDataF->nRef = 1; - - // last - fSet.pLastF = (SLastFile *)taosMemoryCalloc(1, sizeof(SLastFile)); - if (fSet.pLastF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - fSet.pLastF->nRef = 1; - - // sma - fSet.pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile)); - if (fSet.pSmaF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - fSet.pSmaF->nRef = 1; - - n += tGetDFileSet(pData + n, &fSet); + n += nt; if (taosArrayPush(pTsdb->fs.aDFileSet, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -533,8 +509,8 @@ int32_t tsdbFSClose(STsdb *pTsdb) { taosMemoryFree(pSet->pDataF); // last - ASSERT(pSet->pLastF->nRef == 1); - taosMemoryFree(pSet->pLastF); + ASSERT(pSet->aLastF[0]->nRef == 1); + taosMemoryFree(pSet->aLastF[0]); // sma ASSERT(pSet->pSmaF->nRef == 1); @@ -568,7 +544,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid}; + SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nLastF = 1}; // head fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); @@ -586,15 +562,15 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { } *fSet.pDataF = *pSet->pDataF; - // data - fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.pLastF == NULL) { + // last + fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (fSet.aLastF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - *fSet.pLastF = *pSet->pLastF; + *fSet.aLastF[0] = *pSet->aLastF[0]; - // last + // sma fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); if (fSet.pSmaF == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -651,14 +627,14 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { if (c == 0) { *pDFileSet->pHeadF = *pSet->pHeadF; *pDFileSet->pDataF = *pSet->pDataF; - *pDFileSet->pLastF = *pSet->pLastF; + *pDFileSet->aLastF[0] = *pSet->aLastF[0]; *pDFileSet->pSmaF = *pSet->pSmaF; goto _exit; } } - SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid}; + SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nLastF = 1}; // head fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); @@ -676,15 +652,15 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } *fSet.pDataF = *pSet->pDataF; - // data - fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.pLastF == NULL) { + // last + fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (fSet.aLastF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - *fSet.pLastF = *pSet->pLastF; + *fSet.aLastF[0] = *pSet->aLastF[0]; - // last + // sma fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); if (fSet.pSmaF == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -837,24 +813,24 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { } // last - fSet.pLastF = pSetOld->pLastF; - if ((!sameDisk) || (pSetOld->pLastF->commitID != pSetNew->pLastF->commitID)) { - pSetOld->pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (pSetOld->pLastF == NULL) { + fSet.aLastF[0] = pSetOld->aLastF[0]; + if ((!sameDisk) || (pSetOld->aLastF[0]->commitID != pSetNew->aLastF[0]->commitID)) { + pSetOld->aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (pSetOld->aLastF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *pSetOld->pLastF = *pSetNew->pLastF; - pSetOld->pLastF->nRef = 1; + *pSetOld->aLastF[0] = *pSetNew->aLastF[0]; + pSetOld->aLastF[0]->nRef = 1; - nRef = atomic_sub_fetch_32(&fSet.pLastF->nRef, 1); + nRef = atomic_sub_fetch_32(&fSet.aLastF[0]->nRef, 1); if (nRef == 0) { - tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pLastF, fname); + tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.aLastF[0], fname); taosRemoveFile(fname); - taosMemoryFree(fSet.pLastF); + taosMemoryFree(fSet.aLastF[0]); } } else { - ASSERT(pSetOld->pLastF->size == pSetNew->pLastF->size); + ASSERT(pSetOld->aLastF[0]->size == pSetNew->aLastF[0]->size); } // sma @@ -902,11 +878,11 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { taosMemoryFree(pSetOld->pDataF); } - nRef = atomic_sub_fetch_32(&pSetOld->pLastF->nRef, 1); + nRef = atomic_sub_fetch_32(&pSetOld->aLastF[0]->nRef, 1); if (nRef == 0) { - tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pLastF, fname); + tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aLastF[0], fname); taosRemoveFile(fname); - taosMemoryFree(pSetOld->pLastF); + taosMemoryFree(pSetOld->aLastF[0]); } nRef = atomic_sub_fetch_32(&pSetOld->pSmaF->nRef, 1); @@ -922,6 +898,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { _add_new: fSet.diskId = pSetNew->diskId; fSet.fid = pSetNew->fid; + fSet.nLastF = 1; // head fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); @@ -942,13 +919,13 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { fSet.pDataF->nRef = 1; // last - fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.pLastF == NULL) { + fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (fSet.aLastF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *fSet.pLastF = *pSetNew->pLastF; - fSet.pLastF->nRef = 1; + *fSet.aLastF[0] = *pSetNew->aLastF[0]; + fSet.aLastF[0]->nRef = 1; // sma fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); @@ -1002,7 +979,7 @@ int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { nRef = atomic_fetch_add_32(&pSet->pDataF->nRef, 1); ASSERT(nRef > 0); - nRef = atomic_fetch_add_32(&pSet->pLastF->nRef, 1); + nRef = atomic_fetch_add_32(&pSet->aLastF[0]->nRef, 1); ASSERT(nRef > 0); nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1); @@ -1054,12 +1031,12 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { } // last - nRef = atomic_sub_fetch_32(&pSet->pLastF->nRef, 1); + nRef = atomic_sub_fetch_32(&pSet->aLastF[0]->nRef, 1); ASSERT(nRef >= 0); if (nRef == 0) { - tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname); + tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[0], fname); taosRemoveFile(fname); - taosMemoryFree(pSet->pLastF); + taosMemoryFree(pSet->aLastF[0]); } // sma diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 00d2ac848f..4814d793f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -195,8 +195,10 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { n += tPutSmaFile(p ? p + n : p, pSet->pSmaF); // last - n += tPutU8(p ? p + n : p, 1); // for future compatibility - n += tPutLastFile(p ? p + n : p, pSet->pLastF); + n += tPutU8(p ? p + n : p, pSet->nLastF); + for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { + n += tPutLastFile(p ? p + n : p, pSet->aLastF[iLast]); + } return n; } @@ -208,15 +210,40 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { n += tGetI32v(p + n, &pSet->diskId.id); n += tGetI32v(p + n, &pSet->fid); - // data + // head + pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile)); + if (pSet->pHeadF == NULL) { + return -1; + } + pSet->pHeadF->nRef = 1; n += tGetHeadFile(p + n, pSet->pHeadF); + + // data + pSet->pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile)); + if (pSet->pDataF == NULL) { + return -1; + } + pSet->pDataF->nRef = 1; n += tGetDataFile(p + n, pSet->pDataF); + + // sma + pSet->pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile)); + if (pSet->pSmaF == NULL) { + return -1; + } + pSet->pSmaF->nRef = 1; n += tGetSmaFile(p + n, pSet->pSmaF); // last - uint8_t nLast; - n += tGetU8(p + n, &nLast); - n += tGetLastFile(p + n, pSet->pLastF); + n += tGetU8(p + n, &pSet->nLastF); + for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { + pSet->aLastF[iLast] = (SLastFile *)taosMemoryCalloc(1, sizeof(SLastFile)); + if (pSet->aLastF[iLast] == NULL) { + return -1; + } + pSet->aLastF[iLast]->nRef = 1; + n += tGetLastFile(p + n, pSet->aLastF[iLast]); + } return n; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index c8f3862071..48fa80a788 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -436,14 +436,6 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS goto _err; } - // last - tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname); - pReader->pLastFD = taosOpenFile(fname, TD_FILE_READ); - if (pReader->pLastFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - // sma tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); pReader->pSmaFD = taosOpenFile(fname, TD_FILE_READ); @@ -452,6 +444,14 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS goto _err; } + // last + tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[0], fname); + pReader->pLastFD = taosOpenFile(fname, TD_FILE_READ); + if (pReader->pLastFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + *ppReader = pReader; return code; @@ -565,8 +565,8 @@ _err: int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { int32_t code = 0; - int64_t offset = pReader->pSet->pLastF->offset; - int64_t size = pReader->pSet->pLastF->size - offset; + int64_t offset = pReader->pSet->aLastF[0]->offset; + int64_t size = pReader->pSet->aLastF[0]->size - offset; int64_t n; uint32_t delimiter; @@ -935,11 +935,11 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS .fid = pSet->fid, .pHeadF = &pWriter->fHead, .pDataF = &pWriter->fData, - .pLastF = &pWriter->fLast, + .aLastF[0] = &pWriter->fLast, .pSmaF = &pWriter->fSma}; pWriter->fHead = *pSet->pHeadF; pWriter->fData = *pSet->pDataF; - pWriter->fLast = *pSet->pLastF; + pWriter->fLast = *pSet->aLastF[0]; pWriter->fSma = *pSet->pSmaF; // head @@ -1554,8 +1554,8 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { taosCloseFile(&PInFD); // last - tsdbLastFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pLastF, fNameFrom); - tsdbLastFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pLastF, fNameTo); + tsdbLastFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aLastF[0], fNameFrom); + tsdbLastFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aLastF[0], fNameTo); pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { @@ -1569,7 +1569,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { goto _err; } - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pLastF->size); + n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aLastF[0]->size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index a30b9154ab..9fd2c4f6cd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -60,7 +60,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { if (expLevel < 0) { taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->pLastF); + taosMemoryFree(pSet->aLastF[0]); taosMemoryFree(pSet->pSmaF); taosArrayRemove(fs.aDFileSet, iSet); iSet--; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index ab2b2b617a..dc2c2e7b05 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -933,11 +933,12 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 SDataFile fData; SLastFile fLast; SSmaFile fSma; - SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma}; + SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aLastF[0] = &fLast, .pSmaF = &fSma}; if (pSet) { wSet.diskId = pSet->diskId; wSet.fid = fid; + wSet.nLastF = 1; fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fData = *pSet->pDataF; fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; @@ -945,6 +946,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 } else { wSet.diskId = (SDiskID){.level = 0, .id = 0}; wSet.fid = fid; + wSet.nLastF = 1; fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fData = (SDataFile){.commitID = pWriter->commitID, .size = 0}; fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0, .offset = 0}; From 2fff10278f1085626932fb2715181f70101b529f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 23 Aug 2022 09:28:01 +0800 Subject: [PATCH 003/102] more code --- source/dnode/vnode/src/inc/tsdb.h | 9 +++++---- source/dnode/vnode/src/tsdb/tsdbFS.c | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0cac1faa25..230e4ddaad 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -65,10 +65,11 @@ typedef struct SBlockInfo SBlockInfo; typedef struct SSmaInfo SSmaInfo; typedef struct SBlockCol SBlockCol; -#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) -#define TSDB_MAX_SUBBLOCKS 8 -#define TSDB_MAX_LAST_FILE 16 -#define TSDB_FHDR_SIZE 512 +#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) +#define TSDB_MAX_SUBBLOCKS 8 +#define TSDB_MAX_LAST_FILE 16 +#define TSDB_DEFAULT_LAST_FILE 8 +#define TSDB_FHDR_SIZE 512 #define HAS_NONE ((int8_t)0x1) #define HAS_NULL ((int8_t)0x2) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index d86cb7ab9b..4fc2398206 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -562,14 +562,6 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { } *fSet.pDataF = *pSet->pDataF; - // last - fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.aLastF[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *fSet.aLastF[0] = *pSet->aLastF[0]; - // sma fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); if (fSet.pSmaF == NULL) { @@ -582,6 +574,16 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } + + // last + for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { + fSet.aLastF[iLast] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (fSet.aLastF[iLast] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + *fSet.aLastF[iLast] = *pSet->aLastF[iLast]; + } } _exit: From d65956d1a33cac4ed52236fa77f09cbeb8f8fbe6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 23 Aug 2022 10:59:30 +0800 Subject: [PATCH 004/102] more code --- source/dnode/vnode/CMakeLists.txt | 2 ++ source/dnode/vnode/src/tsdb/tsdbCommit.c | 5 ++--- source/dnode/vnode/src/tsdb/tsdbCompact.c | 27 +++++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbMerge.c | 27 +++++++++++++++++++++++ 4 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbCompact.c create mode 100644 source/dnode/vnode/src/tsdb/tsdbMerge.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 0944a6efbc..a3ddad72fc 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -51,6 +51,8 @@ target_sources( "src/tsdb/tsdbRetention.c" "src/tsdb/tsdbDiskData.c" "src/tsdb/tsdbCompress.c" + "src/tsdb/tsdbMerge.c" + "src/tsdb/tsdbCompact.c" # tq "src/tq/tq.c" diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 59afec00ba..bd0a87803c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -35,8 +35,8 @@ typedef struct { int32_t minRow; int32_t maxRow; int8_t cmprAlg; - SArray *aTbDataP; - STsdbFS fs; + SArray *aTbDataP; // memory + STsdbFS fs; // disk // -------------- TSKEY nextKey; // reset by each table commit int32_t commitFid; @@ -1278,7 +1278,6 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - code = tsdbFSCopy(pTsdb, &pCommitter->fs); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c new file mode 100644 index 0000000000..fb3917be64 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbCompact.c @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +typedef struct { + STsdb *pTsdb; + STsdbFS fs; +} STsdbCompactor; + +int32_t tsdbCompact(STsdb *pTsdb) { + int32_t code = 0; + // TODO + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c new file mode 100644 index 0000000000..8b2685e97b --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +typedef struct { + STsdb *pTsdb; + STsdbFS fs; +} STsdbMerger; + +int32_t tsdbMerge(STsdb *pTsdb) { + int32_t code = 0; + // TODO + return code; +} \ No newline at end of file From 478d5ebb649b96ca035af1909f516e88f0495423 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 23 Aug 2022 13:49:10 +0800 Subject: [PATCH 005/102] more code --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 43 +---- source/dnode/vnode/src/tsdb/tsdbFS.c | 39 +++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 151 ++++++++++-------- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 43 ++--- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- 7 files changed, 139 insertions(+), 145 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 230e4ddaad..1300c2ee8f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -262,7 +262,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); -int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); +int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData); int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index b9f3897674..4963754eed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -481,7 +481,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { taosArrayClear(state->aBlockL); } - code = tsdbReadBlockL(state->pDataFReader, state->aBlockL); + code = tsdbReadBlockL(state->pDataFReader, 0, state->aBlockL); if (code) goto _err; // SBlockL *pBlockL = (SBlockL *)taosArrayGet(state->aBlockL, state->iBlockL); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index bd0a87803c..98ad8ea8fa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -35,6 +35,7 @@ typedef struct { int32_t minRow; int32_t maxRow; int8_t cmprAlg; + int8_t maxLast; SArray *aTbDataP; // memory STsdbFS fs; // disk // -------------- @@ -45,19 +46,11 @@ typedef struct { // commit file data struct { SDataFReader *pReader; - // data - SArray *aBlockIdx; // SArray - int32_t iBlockIdx; - SBlockIdx *pBlockIdx; - SMapData mBlock; // SMapData - SBlockData bData; - // last - SArray *aBlockL; // SArray - int32_t iBlockL; - SBlockData bDatal; - int32_t iRow; - SRowInfo *pRowInfo; - SRowInfo rowInfo; + SArray *aBlockIdx; // SArray + int32_t iBlockIdx; + SBlockIdx *pBlockIdx; + SMapData mBlock; // SMapData + SBlockData bData; } dReader; struct { SDataFWriter *pWriter; @@ -437,20 +430,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { pCommitter->dReader.pBlockIdx = NULL; } tBlockDataReset(&pCommitter->dReader.bData); - - // last - code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL); - if (code) goto _err; - - pCommitter->dReader.iBlockL = -1; - pCommitter->dReader.iRow = -1; - pCommitter->dReader.pRowInfo = &pCommitter->dReader.rowInfo; - tBlockDataReset(&pCommitter->dReader.bDatal); - code = tsdbCommitterNextLastRow(pCommitter); - if (code) goto _err; } else { pCommitter->dReader.pBlockIdx = NULL; - pCommitter->dReader.pRowInfo = NULL; } // Writer @@ -1273,6 +1254,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; + pCommitter->maxLast = TSDB_DEFAULT_LAST_FILE; // TODO: make it as a config pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); if (pCommitter->aTbDataP == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1301,15 +1283,6 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { code = tBlockDataCreate(&pCommitter->dReader.bData); if (code) goto _exit; - pCommitter->dReader.aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pCommitter->dReader.aBlockL == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - code = tBlockDataCreate(&pCommitter->dReader.bDatal); - if (code) goto _exit; - // Writer pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCommitter->dWriter.aBlockIdx == NULL) { @@ -1338,8 +1311,6 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { taosArrayDestroy(pCommitter->dReader.aBlockIdx); tMapDataClear(&pCommitter->dReader.mBlock); tBlockDataDestroy(&pCommitter->dReader.bData, 1); - taosArrayDestroy(pCommitter->dReader.aBlockL); - tBlockDataDestroy(&pCommitter->dReader.bDatal, 1); // Writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 4fc2398206..000e262b92 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -544,7 +544,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nLastF = 1}; + SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid}; // head fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); @@ -576,13 +576,13 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { } // last - for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { - fSet.aLastF[iLast] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.aLastF[iLast] == NULL) { + for (fSet.nLastF = 0; fSet.nLastF < pSet->nLastF; fSet.nLastF++) { + fSet.aLastF[fSet.nLastF] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (fSet.aLastF[fSet.nLastF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - *fSet.aLastF[iLast] = *pSet->aLastF[iLast]; + *fSet.aLastF[fSet.nLastF] = *pSet->aLastF[fSet.nLastF]; } } @@ -981,12 +981,14 @@ int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { nRef = atomic_fetch_add_32(&pSet->pDataF->nRef, 1); ASSERT(nRef > 0); - nRef = atomic_fetch_add_32(&pSet->aLastF[0]->nRef, 1); - ASSERT(nRef > 0); - nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1); ASSERT(nRef > 0); + for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { + nRef = atomic_fetch_add_32(&pSet->aLastF[iLast]->nRef, 1); + ASSERT(nRef > 0); + } + if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -1032,15 +1034,6 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { taosMemoryFree(pSet->pDataF); } - // last - nRef = atomic_sub_fetch_32(&pSet->aLastF[0]->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[0], fname); - taosRemoveFile(fname); - taosMemoryFree(pSet->aLastF[0]); - } - // sma nRef = atomic_sub_fetch_32(&pSet->pSmaF->nRef, 1); ASSERT(nRef >= 0); @@ -1049,6 +1042,18 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { taosRemoveFile(fname); taosMemoryFree(pSet->pSmaF); } + + // last + for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { + nRef = atomic_sub_fetch_32(&pSet->aLastF[iLast]->nRef, 1); + ASSERT(nRef >= 0); + if (nRef == 0) { + tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[iLast], fname); + taosRemoveFile(fname); + taosMemoryFree(pSet->aLastF[iLast]); + /* code */ + } + } } taosArrayDestroy(pFS->aDFileSet); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a4738781f5..abe83ccb4b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -16,7 +16,7 @@ #include "osDef.h" #include "tsdb.h" -#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define ALL_ROWS_CHECKED_INDEX (INT16_MIN) #define DEFAULT_ROW_INDEX_VAL (-1) @@ -40,15 +40,15 @@ typedef struct { typedef struct STableBlockScanInfo { uint64_t uid; TSKEY lastKey; - SMapData mapData; // block info (compressed) - SArray* pBlockList; // block data index list - SIterInfo iter; // mem buffer skip list iterator - SIterInfo iiter; // imem buffer skip list iterator - SArray* delSkyline; // delete info for this table - int32_t fileDelIndex; // file block delete index - int32_t lastBlockDelIndex;// delete index for last block - bool iterInit; // whether to initialize the in-memory skip list iterator or not - int16_t indexInBlockL;// row position in last block + SMapData mapData; // block info (compressed) + SArray* pBlockList; // block data index list + SIterInfo iter; // mem buffer skip list iterator + SIterInfo iiter; // imem buffer skip list iterator + SArray* delSkyline; // delete info for this table + int32_t fileDelIndex; // file block delete index + int32_t lastBlockDelIndex; // delete index for last block + bool iterInit; // whether to initialize the in-memory skip list iterator or not + int16_t indexInBlockL; // row position in last block } STableBlockScanInfo; typedef struct SBlockOrderWrapper { @@ -96,15 +96,15 @@ typedef struct SLastBlockReader { SVersionRange verRange; int32_t order; uint64_t uid; - int16_t* rowIndex; // row index ptr, usually from the STableBlockScanInfo->indexInBlockL + int16_t* rowIndex; // row index ptr, usually from the STableBlockScanInfo->indexInBlockL } SLastBlockReader; typedef struct SFilesetIter { - int32_t numOfFiles; // number of total files - int32_t index; // current accessed index in the list - SArray* pFileList; // data file list + int32_t numOfFiles; // number of total files + int32_t index; // current accessed index in the list + SArray* pFileList; // data file list int32_t order; - SLastBlockReader* pLastBlockReader; // last file block reader + SLastBlockReader* pLastBlockReader; // last file block reader } SFilesetIter; typedef struct SFileDataBlockInfo { @@ -116,9 +116,9 @@ typedef struct SFileDataBlockInfo { typedef struct SDataBlockIter { int32_t numOfBlocks; int32_t index; - SArray* blockList; // SArray + SArray* blockList; // SArray int32_t order; - SBlock block; // current SBlock data + SBlock block; // current SBlock data SHashObj* pTableMap; } SDataBlockIter; @@ -169,12 +169,13 @@ static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i static TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader, SRowMerger* pMerger); -static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger); +static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, + SRowMerger* pMerger); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, - int32_t rowIndex); + int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); @@ -187,9 +188,9 @@ static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdb static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, int8_t* pLevel); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); -static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader); -static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); -static int32_t doBuildDataBlock(STsdbReader* pReader); +static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader); +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); +static int32_t doBuildDataBlock(STsdbReader* pReader); static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; @@ -320,7 +321,8 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap } // init file iterator -static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdbReader* pReader/*int32_t order, const char* idstr*/) { +static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, + STsdbReader* pReader /*int32_t order, const char* idstr*/) { size_t numOfFileset = taosArrayGetSize(aDFileSet); pIter->index = ASCENDING_TRAVERSE(pReader->order) ? -1 : numOfFileset; @@ -338,8 +340,8 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb SLastBlockReader* pLReader = pIter->pLastBlockReader; pLReader->pBlockL = taosArrayInit(4, sizeof(SBlockL)); - pLReader->order = pReader->order; - pLReader->window = pReader->window; + pLReader->order = pReader->order; + pLReader->window = pReader->window; pLReader->verRange = pReader->verRange; pLReader->currentBlockIndex = -1; @@ -658,7 +660,7 @@ static void cleanupTableScanInfo(SHashObj* pTableMap) { } static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* pLastBlockIndex, - SBlockNumber * pBlockNum, SArray* pQualifiedLastBlock) { + SBlockNumber* pBlockNum, SArray* pQualifiedLastBlock) { int32_t numOfQTable = 0; size_t sizeInDisk = 0; size_t numOfTables = taosArrayGetSize(pIndexList); @@ -704,7 +706,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* } size_t numOfLast = taosArrayGetSize(pLastBlockIndex); - for(int32_t i = 0; i < numOfLast; ++i) { + for (int32_t i = 0; i < numOfLast; ++i) { SBlockL* pLastBlock = taosArrayGet(pLastBlockIndex, i); if (pLastBlock->suid != pReader->suid) { continue; @@ -729,9 +731,10 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* int32_t total = pBlockNum->numOfLastBlocks + pBlockNum->numOfBlocks; double el = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("load block of %d tables completed, blocks:%d in %d tables, lastBlock:%d, size:%.2f Kb, elapsed time:%.2f ms %s", - numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastBlocks, sizeInDisk - / 1000.0, el, pReader->idStr); + tsdbDebug( + "load block of %d tables completed, blocks:%d in %d tables, lastBlock:%d, size:%.2f Kb, elapsed time:%.2f ms %s", + numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastBlocks, sizeInDisk / 1000.0, el, + pReader->idStr); pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; @@ -879,7 +882,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI elapsedTime = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug("%p load file block into buffer, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 - ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", + ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); } else { @@ -977,10 +980,10 @@ static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, v } static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) { - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); + SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); if (pFBlock != NULL) { STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); - int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx); + int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx); tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetBlock); } @@ -1097,8 +1100,8 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } int64_t et = taosGetTimestampUs(); - tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks, (et - st) / 1000.0, - pReader->idStr); + tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks, + (et - st) / 1000.0, pReader->idStr); cleanupBlockOrderSupporter(&sup); taosMemoryFree(pTree); @@ -1301,7 +1304,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc // todo here we need to each key in the last files to identify if it is really overlapped with last block bool overlapWithlastBlock = false; if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0 && (pLastBlockReader->currentBlockIndex != -1)) { - SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); + SBlockL* pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); } @@ -1395,7 +1398,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* } static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, - SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { + SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; SBlockData* pBlockData = &pReader->status.fileBlockData; @@ -1406,14 +1409,14 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf tsLast = getCurrentKeyInLastBlock(pLastBlockReader); } - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; int64_t minKey = 0; if (pReader->order == TSDB_ORDER_ASC) { - minKey = INT64_MAX; // chosen the minimum value + minKey = INT64_MAX; // chosen the minimum value if (minKey > tsLast && pLastBlockReader->lastBlockData.nRow > 0) { minKey = tsLast; } @@ -1443,7 +1446,7 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf bool init = false; // ASC: file block ---> last block -----> imem -----> mem - //DESC: mem -----> imem -----> last block -----> file block + // DESC: mem -----> imem -----> last block -----> file block if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; @@ -1583,7 +1586,8 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } -static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { +static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, + SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; @@ -1595,7 +1599,7 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo ASSERT(pRow != NULL && piRow != NULL); SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t tsLast = INT64_MIN; + int64_t tsLast = INT64_MIN; if (hasDataInLastBlock(pLastBlockReader)) { tsLast = getCurrentKeyInLastBlock(pLastBlockReader); } @@ -1605,7 +1609,7 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); - int64_t minKey = 0;//INT64_MAX; + int64_t minKey = 0; // INT64_MAX; if (ASCENDING_TRAVERSE(pReader->order)) { minKey = INT64_MAX; // let's find the minimum if (minKey > k.ts) { @@ -1624,7 +1628,7 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo minKey = tsLast; } } else { - minKey = INT64_MIN; // let find the maximum ts value + minKey = INT64_MIN; // let find the maximum ts value if (minKey < k.ts) { minKey = k.ts; } @@ -1734,7 +1738,8 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo return TSDB_CODE_SUCCESS; } -static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { +static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, + SBlockData* pBlockData) { SRowMerger merge = {0}; STSRow* pTSRow = NULL; @@ -1921,11 +1926,11 @@ static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid } } -static void setAllRowsChecked(SLastBlockReader *pLastBlockReader) { +static void setAllRowsChecked(SLastBlockReader* pLastBlockReader) { *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; } -static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { +static bool nextRowInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { int32_t step = (pLastBlockReader->order == TSDB_ORDER_ASC) ? 1 : -1; if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { return false; @@ -1934,7 +1939,7 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader, STableBlockSc *(pLastBlockReader->rowIndex) += step; SBlockData* pBlockData = &pLastBlockReader->lastBlockData; - for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow && i >= 0; i += step) { + for (int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow && i >= 0; i += step) { if (pBlockData->aUid != NULL && pBlockData->aUid[i] != pLastBlockReader->uid) { continue; } @@ -1990,7 +1995,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t key = (pBlockData->nRow > 0)? pBlockData->aTSKEY[pDumpInfo->rowIndex]:INT64_MIN; + int64_t key = (pBlockData->nRow > 0) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); @@ -2071,7 +2076,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } } else { // desc order SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); STSRow* pTSRow = NULL; SRowMerger merge = {0}; @@ -2114,7 +2119,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI static int32_t buildComposedDataBlock(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->pResBlock; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo != NULL) { @@ -2178,7 +2183,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { setComposedBlockFlag(pReader, true); int64_t et = taosGetTimestampUs(); - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%d, elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 + " rows:%d, elapsed time:%.2f ms %s", pReader, pBlockScanInfo->uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, (et - st) / 1000.0, pReader->idStr); @@ -2376,7 +2382,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return code; } - code = tsdbReadBlockL(pReader->pFileReader, pLastBlocks); + code = tsdbReadBlockL(pReader->pFileReader, 0, pLastBlocks); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; @@ -2411,12 +2417,13 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return TSDB_CODE_SUCCESS; } -static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo *pBlockScanInfo, STsdbReader* pReader) { +static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader) { SArray* pBlocks = pLastBlockReader->pBlockL; SBlockL* pBlock = NULL; uint64_t uid = pBlockScanInfo->uid; - int32_t totalLastBlocks = (int32_t)taosArrayGetSize(pBlocks); + int32_t totalLastBlocks = (int32_t)taosArrayGetSize(pBlocks); initMemDataIterator(pBlockScanInfo, pReader); @@ -2443,7 +2450,8 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STable } int64_t st = taosGetTimestampUs(); - int32_t code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); + int32_t code = + tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p init block data failed, code:%s %s", pReader, tstrerror(code), pReader->idStr); return code; @@ -2471,10 +2479,10 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STable } static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; + SReaderStatus* pStatus = &pReader->status; SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; - while(1) { + while (1) { if (pStatus->pTableIter == NULL) { pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); if (pStatus->pTableIter == NULL) { @@ -2485,7 +2493,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // load the last data block of current table // todo opt perf by avoiding load last block repeatly STableBlockScanInfo* pScanInfo = pStatus->pTableIter; - int32_t code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); + int32_t code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2560,7 +2568,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // note: the lastblock may be null here initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { + if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || + pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); } } @@ -2591,11 +2600,11 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { // only return the rows in last block int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); - ASSERT (tsLast >= pBlock->maxKey.ts); + ASSERT(tsLast >= pBlock->maxKey.ts); tBlockDataReset(&pReader->status.fileBlockData); code = buildComposedDataBlock(pReader); - } else { // whole block is required, return it directly + } else { // whole block is required, return it directly SDataBlockInfo* pInfo = &pReader->pResBlock->info; pInfo->rows = pBlock->nRow; pInfo->uid = pScanInfo->uid; @@ -2670,7 +2679,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl // initialize the block iterator for a new fileset if (num.numOfBlocks > 0) { code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks); - } else { // no block data, only last block exists + } else { // no block data, only last block exists tBlockDataReset(&pReader->status.fileBlockData); resetDataBlockIterator(pBlockIter, pReader->order, pReader->status.pTableMap); } @@ -2695,7 +2704,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { SDataBlockIter* pBlockIter = &pReader->status.blockIter; if (pBlockIter->numOfBlocks == 0) { - _begin: + _begin: code = doLoadLastBlockSequentially(pReader); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2742,7 +2751,8 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { bool hasNext = blockIteratorNext(&pReader->status.blockIter); if (hasNext) { // check for the next block in the block accessed order list initBlockDumpInfo(pReader, pBlockIter); - } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0) { // data blocks in current file are exhausted, let's try the next file now + } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > + 0) { // data blocks in current file are exhausted, let's try the next file now tBlockDataReset(&pReader->status.fileBlockData); resetDataBlockIterator(pBlockIter, pReader->order, pReader->status.pTableMap); goto _begin; @@ -3101,8 +3111,9 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc } // todo check if the rows are dropped or not -int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger) { - while(nextRowInLastBlock(pLastBlockReader, pScanInfo)) { +int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, + SRowMerger* pMerger) { + while (nextRowInLastBlock(pLastBlockReader, pScanInfo)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, *pLastBlockReader->rowIndex); @@ -3287,7 +3298,8 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex) { +int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, + int32_t rowIndex) { int32_t i = 0, j = 0; int32_t outputRowIndex = pResBlock->info.rows; @@ -3385,7 +3397,6 @@ void* tsdbGetIvtIdx(SMeta* pMeta) { uint64_t getReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; } - // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTableList, STsdbReader** ppReader, const char* idstr) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 48fa80a788..0aa32702dd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -399,8 +399,8 @@ struct SDataFReader { SDFileSet *pSet; TdFilePtr pHeadFD; TdFilePtr pDataFD; - TdFilePtr pLastFD; TdFilePtr pSmaFD; + TdFilePtr aLastFD[TSDB_MAX_LAST_FILE]; uint8_t *aBuf[3]; }; @@ -445,11 +445,13 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS } // last - tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[0], fname); - pReader->pLastFD = taosOpenFile(fname, TD_FILE_READ); - if (pReader->pLastFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { + tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[iLast], fname); + pReader->aLastFD[iLast] = taosOpenFile(fname, TD_FILE_READ); + if (pReader->aLastFD[iLast] == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } } *ppReader = pReader; @@ -465,30 +467,35 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { int32_t code = 0; if (*ppReader == NULL) goto _exit; + // head if (taosCloseFile(&(*ppReader)->pHeadFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } + // data if (taosCloseFile(&(*ppReader)->pDataFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (taosCloseFile(&(*ppReader)->pLastFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - + // sma if (taosCloseFile(&(*ppReader)->pSmaFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } + // last + for (int32_t iLast = 0; iLast < (*ppReader)->pSet->nLastF; iLast++) { + if (taosCloseFile(&(*ppReader)->aLastFD[iLast]) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } + for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) { tFree((*ppReader)->aBuf[iBuf]); } - taosMemoryFree(*ppReader); _exit: @@ -563,10 +570,10 @@ _err: return code; } -int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { +int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL) { int32_t code = 0; - int64_t offset = pReader->pSet->aLastF[0]->offset; - int64_t size = pReader->pSet->aLastF[0]->size - offset; + int64_t offset = pReader->pSet->aLastF[iLast]->offset; + int64_t size = pReader->pSet->aLastF[iLast]->size - offset; int64_t n; uint32_t delimiter; @@ -580,13 +587,13 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { if (code) goto _err; // seek - if (taosLSeekFile(pReader->pLastFD, offset, SEEK_SET) < 0) { + if (taosLSeekFile(pReader->aLastFD[iLast], offset, SEEK_SET) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } // read - n = taosReadFile(pReader->pLastFD, pReader->aBuf[0], size); + n = taosReadFile(pReader->aLastFD[iLast], pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -745,7 +752,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo tBlockDataClear(pBlockData); - TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; + TdFilePtr pFD = fromLast ? pReader->aLastFD[0] : pReader->pDataFD; // (todo) // uid + version + tskey code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->aBuf[0], pBlkInfo->szKey, 1); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index dc2c2e7b05..fa7d370583 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -64,7 +64,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); if (code) goto _err; - code = tsdbReadBlockL(pReader->pDataFReader, pReader->aBlockL); + code = tsdbReadBlockL(pReader->pDataFReader, 0, pReader->aBlockL); if (code) goto _err; // init @@ -911,7 +911,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx); if (code) goto _err; - code = tsdbReadBlockL(pWriter->pDataFReader, pWriter->aBlockL); + code = tsdbReadBlockL(pWriter->pDataFReader, 0, pWriter->aBlockL); if (code) goto _err; } else { ASSERT(pWriter->pDataFReader == NULL); From fa257f50cefb36378e459a25ac5979824c1aea7f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 23 Aug 2022 15:47:42 +0800 Subject: [PATCH 006/102] more code --- source/dnode/vnode/src/inc/tsdb.h | 4 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 38 ++++-- source/dnode/vnode/src/tsdb/tsdbFS.c | 40 ++++-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 114 ++++++++---------- 4 files changed, 110 insertions(+), 86 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 1300c2ee8f..4c8208faa7 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -589,13 +589,13 @@ struct SDataFWriter { TdFilePtr pHeadFD; TdFilePtr pDataFD; - TdFilePtr pLastFD; TdFilePtr pSmaFD; + TdFilePtr pLastFD; SHeadFile fHead; SDataFile fData; - SLastFile fLast; SSmaFile fSma; + SLastFile fLast[TSDB_MAX_LAST_FILE]; uint8_t *aBuf[4]; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 98ad8ea8fa..8542eca8c9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -437,31 +437,43 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { // Writer SHeadFile fHead; SDataFile fData; - SLastFile fLast; SSmaFile fSma; - SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aLastF[0] = &fLast, .pSmaF = &fSma}; + SLastFile fLast; + SDFileSet wSet = {0}; if (pRSet) { + ASSERT(pRSet->nLastF < pCommitter->maxLast); + fHead = (SHeadFile){.commitID = pCommitter->commitID}; + fData = *pRSet->pDataF; + fSma = *pRSet->pSmaF; + fLast = (SLastFile){.commitID = pCommitter->commitID}; + wSet.diskId = pRSet->diskId; wSet.fid = pCommitter->commitFid; - wSet.nLastF = 1; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; - fData = *pRSet->pDataF; - fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; - fSma = *pRSet->pSmaF; + wSet.pHeadF = &fHead; + wSet.pDataF = &fData; + wSet.pSmaF = &fSma; + for (int8_t iLast = 0; iLast < pRSet->nLastF; iLast++) { + wSet.aLastF[iLast] = pRSet->aLastF[iLast]; + } + wSet.nLastF = pRSet->nLastF + 1; + wSet.aLastF[wSet.nLastF - 1] = &fLast; // todo } else { + fHead = (SHeadFile){.commitID = pCommitter->commitID}; + fData = (SDataFile){.commitID = pCommitter->commitID}; + fSma = (SSmaFile){.commitID = pCommitter->commitID}; + fLast = (SLastFile){.commitID = pCommitter->commitID}; + SDiskID did = {0}; - tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did); - tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); wSet.diskId = did; wSet.fid = pCommitter->commitFid; + wSet.pHeadF = &fHead; + wSet.pDataF = &fData; + wSet.pSmaF = &fSma; wSet.nLastF = 1; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; - fData = (SDataFile){.commitID = pCommitter->commitID, .size = 0}; - fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; - fSma = (SSmaFile){.commitID = pCommitter->commitID, .size = 0}; + wSet.aLastF[0] = &fLast; } code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 000e262b92..e6a90825ab 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -629,13 +629,35 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { if (c == 0) { *pDFileSet->pHeadF = *pSet->pHeadF; *pDFileSet->pDataF = *pSet->pDataF; - *pDFileSet->aLastF[0] = *pSet->aLastF[0]; *pDFileSet->pSmaF = *pSet->pSmaF; + // last + if (pSet->nLastF > pDFileSet->nLastF) { + ASSERT(pSet->nLastF == pDFileSet->nLastF + 1); + + pDFileSet->aLastF[pDFileSet->nLastF] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (pDFileSet->aLastF[pDFileSet->nLastF] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + *pDFileSet->aLastF[pDFileSet->nLastF] = *pSet->aLastF[pSet->nLastF - 1]; + pDFileSet->nLastF++; + } else if (pSet->nLastF < pDFileSet->nLastF) { + ASSERT(pSet->nLastF == 1); + for (int32_t iLast = 1; iLast < pDFileSet->nLastF; iLast++) { + taosMemoryFree(pDFileSet->aLastF[iLast]); + } + + *pDFileSet->aLastF[0] = *pSet->aLastF[0]; + pDFileSet->nLastF = 1; + } else { + ASSERT(0); + } goto _exit; } } + ASSERT(pSet->nLastF == 1); SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nLastF = 1}; // head @@ -654,14 +676,6 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } *fSet.pDataF = *pSet->pDataF; - // last - fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.aLastF[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *fSet.aLastF[0] = *pSet->aLastF[0]; - // sma fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); if (fSet.pSmaF == NULL) { @@ -670,6 +684,14 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } *fSet.pSmaF = *pSet->pSmaF; + // last + fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (fSet.aLastF[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + *fSet.aLastF[0] = *pSet->aLastF[0]; + if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 0aa32702dd..e07b0e8f94 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -936,18 +936,22 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - if (code) goto _err; pWriter->pTsdb = pTsdb; - pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, - .fid = pSet->fid, - .pHeadF = &pWriter->fHead, - .pDataF = &pWriter->fData, - .aLastF[0] = &pWriter->fLast, - .pSmaF = &pWriter->fSma}; + pWriter->wSet = (SDFileSet){ + .diskId = pSet->diskId, + .fid = pSet->fid, + .pHeadF = &pWriter->fHead, + .pDataF = &pWriter->fData, + .pSmaF = &pWriter->fSma, + .nLastF = pSet->nLastF // + }; pWriter->fHead = *pSet->pHeadF; pWriter->fData = *pSet->pDataF; - pWriter->fLast = *pSet->aLastF[0]; pWriter->fSma = *pSet->pSmaF; + for (int8_t iLast = 0; iLast < pSet->nLastF; iLast++) { + pWriter->wSet.aLastF[iLast] = &pWriter->fLast[iLast]; + pWriter->fLast[iLast] = *pSet->aLastF[iLast]; + } // head flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; @@ -998,36 +1002,6 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS ASSERT(n == pWriter->fData.size); } - // last - if (pWriter->fLast.size == 0) { - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_WRITE; - } - tsdbLastFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fLast, fname); - pWriter->pLastFD = taosOpenFile(fname, flag); - if (pWriter->pLastFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - if (pWriter->fLast.size == 0) { - n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pWriter->fLast.size += TSDB_FHDR_SIZE; - } else { - n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_END); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - ASSERT(n == pWriter->fLast.size); - } - // sma if (pWriter->fSma.size == 0) { flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; @@ -1058,6 +1032,22 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS ASSERT(n == pWriter->fSma.size); } + // last + ASSERT(pWriter->fLast[pSet->nLastF - 1].size == 0); + flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + tsdbLastFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fLast[pSet->nLastF - 1], fname); + pWriter->pLastFD = taosOpenFile(fname, flag); + if (pWriter->pLastFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + pWriter->fLast[pWriter->wSet.nLastF - 1].size += TSDB_FHDR_SIZE; + *ppWriter = pWriter; return code; @@ -1085,12 +1075,12 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { goto _err; } - if (taosFsyncFile((*ppWriter)->pLastFD) < 0) { + if (taosFsyncFile((*ppWriter)->pSmaFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (taosFsyncFile((*ppWriter)->pSmaFD) < 0) { + if (taosFsyncFile((*ppWriter)->pLastFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -1106,12 +1096,12 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { goto _err; } - if (taosCloseFile(&(*ppWriter)->pLastFD) < 0) { + if (taosCloseFile(&(*ppWriter)->pSmaFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (taosCloseFile(&(*ppWriter)->pSmaFD) < 0) { + if (taosCloseFile(&(*ppWriter)->pLastFD) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -1168,23 +1158,6 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { goto _err; } - // last ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutLastFile(hdr, &pWriter->fLast); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - - n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - // sma ============== memset(hdr, 0, TSDB_FHDR_SIZE); tPutSmaFile(hdr, &pWriter->fSma); @@ -1202,6 +1175,23 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { goto _err; } + // last ============== + memset(hdr, 0, TSDB_FHDR_SIZE); + tPutLastFile(hdr, &pWriter->fLast[pWriter->wSet.nLastF - 1]); + taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); + + n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + return code; _err: @@ -1309,7 +1299,7 @@ _err: int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { int32_t code = 0; - SLastFile *pLastFile = &pWriter->fLast; + SLastFile *pLastFile = &pWriter->fLast[pWriter->wSet.nLastF - 1]; int64_t size; int64_t n; @@ -1437,7 +1427,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock ASSERT(pBlockData->nRow > 0); - pBlkInfo->offset = toLast ? pWriter->fLast.size : pWriter->fData.size; + pBlkInfo->offset = toLast ? pWriter->fLast[pWriter->wSet.nLastF - 1].size : pWriter->fData.size; pBlkInfo->szBlock = 0; pBlkInfo->szKey = 0; @@ -1481,7 +1471,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock // update info if (toLast) { - pWriter->fLast.size += pBlkInfo->szBlock; + pWriter->fLast[pWriter->wSet.nLastF - 1].size += pBlkInfo->szBlock; } else { pWriter->fData.size += pBlkInfo->szBlock; } From 23e82ff68ba7d6c1a1059292296b4f69cc974433 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 23 Aug 2022 16:32:44 +0800 Subject: [PATCH 007/102] more code --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbCommit.c | 132 ++++++++--------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 16 +++ 3 files changed, 59 insertions(+), 90 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4c8208faa7..c07bf2d6c8 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -172,6 +172,7 @@ int32_t tGetDelData(uint8_t *p, void *ph); void tMapDataReset(SMapData *pMapData); void tMapDataClear(SMapData *pMapData); int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(uint8_t *, void *)); +int32_t tMapDataCopy(SMapData *pFrom, SMapData *pTo); void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *)); int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *), int32_t (*tItemCmprFn)(const void *, const void *), void *pItem); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8542eca8c9..08688b442b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -591,7 +591,7 @@ _err: return code; } -static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { +static int32_t tsdbMergeCommitDataBlock(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { int32_t code = 0; STbData *pTbData = pIter->pTbData; SBlockData *pBlockDataR = &pCommitter->dReader.bData; @@ -941,25 +941,9 @@ _err: return code; } -static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { +static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter) { int32_t code = 0; - ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0); - ASSERT(pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, pTbData) >= 0); - - // merge commit table data - STbDataIter iter = {0}; - STbDataIter *pIter = &iter; - TSDBROW *pRow; - - tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - - if (pRow == NULL) goto _exit; - int32_t iBlock = 0; SBlock block; SBlock *pBlock = █ @@ -1010,7 +994,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { code = tsdbMergeAsSubBlock(pCommitter, pIter, pBlock); if (code) goto _err; } else { - code = tsdbMergeCommitData(pCommitter, pIter, pBlock); + code = tsdbMergeCommitDataBlock(pCommitter, pIter, pBlock); if (code) goto _err; } @@ -1041,11 +1025,46 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { } } - // .data append and .last merge - code = tsdbMergeCommitLast(pCommitter, pIter); +_exit: + return code; + +_err: + return code; +} + +static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { + int32_t code = 0; + + ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0); + + // merge commit table data + STbDataIter iter = {0}; + TSDBROW *pRow; + + tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, &iter); + pRow = tsdbTbDataIterGet(&iter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; + } + + if (pRow == NULL) { + if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) == 0) { + code = tMapDataCopy(&pCommitter->dReader.mBlock, &pCommitter->dWriter.mBlock); + if (code) goto _err; + } + + goto _exit; + } + + // commit data + code = tsdbMergeCommitData(pCommitter, &iter); if (code) goto _err; - // end + // commit last + code = tsdbMergeCommitLast(pCommitter, &iter); + if (code) goto _err; + +_exit: if (pCommitter->dWriter.mBlock.nItem > 0) { SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid}; code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); @@ -1056,9 +1075,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { goto _err; } } - -_exit: - pRow = tsdbTbDataIterGet(pIter); + pRow = tsdbTbDataIterGet(&iter); if (pRow) { pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); } @@ -1126,71 +1143,6 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { if (code) goto _err; } - // .last - while (true) { - if (pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, &toTable) >= 0) break; - - SBlockData *pBlockDataR = &pCommitter->dReader.bDatal; - SBlockData *pBlockDataW = &pCommitter->dWriter.bDatal; - tb_uid_t suid = pCommitter->dReader.pRowInfo->suid; - tb_uid_t uid = pCommitter->dReader.pRowInfo->uid; - - ASSERT((pBlockDataR->suid && !pBlockDataR->uid) || (!pBlockDataR->suid && pBlockDataR->uid)); - ASSERT(pBlockDataR->nRow > 0); - - // commit and reset block data schema if need - if (pBlockDataW->suid || pBlockDataW->uid) { - if (pBlockDataW->suid != suid || pBlockDataW->suid == 0) { - if (pBlockDataW->nRow > 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } - tBlockDataReset(pBlockDataW); - } - } - - // set block data schema if need - if (pBlockDataW->suid == 0 && pBlockDataW->uid == 0) { - code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid); - if (code) goto _err; - - code = tBlockDataInit(pBlockDataW, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; - } - - // check if it can make sure that one table data in one block - int32_t nRow = 0; - if (pBlockDataR->suid) { - int32_t iRow = pCommitter->dReader.iRow; - while ((iRow < pBlockDataR->nRow) && (pBlockDataR->aUid[iRow] == uid)) { - nRow++; - iRow++; - } - } else { - ASSERT(pCommitter->dReader.iRow == 0); - nRow = pBlockDataR->nRow; - } - - ASSERT(nRow > 0 && nRow < pCommitter->minRow); - - if (pBlockDataW->nRow + nRow > pCommitter->maxRow) { - ASSERT(pBlockDataW->nRow > 0); - - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } - - while (nRow > 0) { - code = tBlockDataAppendRow(pBlockDataW, &pCommitter->dReader.pRowInfo->row, NULL, uid); - if (code) goto _err; - - code = tsdbCommitterNextLastRow(pCommitter); - if (code) goto _err; - - nRow--; - } - } - return code; _err: diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 6db9d5e6f4..49778542e3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -51,6 +51,22 @@ _exit: return code; } +int32_t tMapDataCopy(SMapData *pFrom, SMapData *pTo) { + int32_t code = 0; + + pTo->nItem = pFrom->nItem; + pTo->nData = pFrom->nData; + code = tRealloc((uint8_t **)&pTo->aOffset, sizeof(int32_t) * pFrom->nItem); + if (code) goto _exit; + code = tRealloc(&pTo->pData, pFrom->nData); + if (code) goto _exit; + memcpy(pTo->aOffset, pFrom->aOffset, sizeof(int32_t) * pFrom->nItem); + memcpy(pTo->pData, pFrom->pData, pFrom->nData); + +_exit: + return code; +} + int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *), int32_t (*tItemCmprFn)(const void *, const void *), void *pItem) { int32_t code = 0; From f43bae0df064d97cf68ff8e1974cc61eab72c4c2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 23 Aug 2022 17:11:17 +0800 Subject: [PATCH 008/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 128 +++++++++++------------ 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 08688b442b..4a31efdc73 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -327,53 +327,53 @@ _exit: return code; } -static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { - int32_t code = 0; +// static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { +// int32_t code = 0; - ASSERT(pCommitter->dReader.pReader); - ASSERT(pCommitter->dReader.pRowInfo); +// ASSERT(pCommitter->dReader.pReader); +// ASSERT(pCommitter->dReader.pRowInfo); - SBlockData *pBlockDatal = &pCommitter->dReader.bDatal; - pCommitter->dReader.iRow++; - if (pCommitter->dReader.iRow < pBlockDatal->nRow) { - if (pBlockDatal->uid) { - pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; - } else { - pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[pCommitter->dReader.iRow]; - } - pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); - } else { - pCommitter->dReader.iBlockL++; - if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - SBlockL *pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - int64_t suid = pBlockL->suid; - int64_t uid = pBlockL->maxUid; +// SBlockData *pBlockDatal = &pCommitter->dReader.bDatal; +// pCommitter->dReader.iRow++; +// if (pCommitter->dReader.iRow < pBlockDatal->nRow) { +// if (pBlockDatal->uid) { +// pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; +// } else { +// pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[pCommitter->dReader.iRow]; +// } +// pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); +// } else { +// pCommitter->dReader.iBlockL++; +// if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { +// SBlockL *pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); +// int64_t suid = pBlockL->suid; +// int64_t uid = pBlockL->maxUid; - code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid); - if (code) goto _exit; +// code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid); +// if (code) goto _exit; - code = tBlockDataInit(pBlockDatal, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); - if (code) goto _exit; +// code = tBlockDataInit(pBlockDatal, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); +// if (code) goto _exit; - code = tsdbReadLastBlock(pCommitter->dReader.pReader, pBlockL, pBlockDatal); - if (code) goto _exit; +// code = tsdbReadLastBlock(pCommitter->dReader.pReader, pBlockL, pBlockDatal); +// if (code) goto _exit; - pCommitter->dReader.iRow = 0; - pCommitter->dReader.pRowInfo->suid = pBlockDatal->suid; - if (pBlockDatal->uid) { - pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; - } else { - pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[0]; - } - pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); - } else { - pCommitter->dReader.pRowInfo = NULL; - } - } +// pCommitter->dReader.iRow = 0; +// pCommitter->dReader.pRowInfo->suid = pBlockDatal->suid; +// if (pBlockDatal->uid) { +// pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; +// } else { +// pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[0]; +// } +// pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); +// } else { +// pCommitter->dReader.pRowInfo = NULL; +// } +// } -_exit: - return code; -} +// _exit: +// return code; +// } static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { int32_t code = 0; @@ -779,7 +779,8 @@ _err: } static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { - int32_t code = 0; + int32_t code = 0; +#if 0 STbData *pTbData = pIter->pTbData; int32_t nRow = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); @@ -938,54 +939,47 @@ _exit: _err: tsdbError("vgId:%d tsdb merge commit last failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +#endif return code; } static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter) { - int32_t code = 0; + int32_t code = 0; + STbData *pTbData = pIter->pTbData; + int32_t iBlock = 0; + SBlock block; + SBlock *pBlock = █ + TSDBROW *pRow = tsdbTbDataIterGet(pIter); - int32_t iBlock = 0; - SBlock block; - SBlock *pBlock = █ if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); } else { pBlock = NULL; } - code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid); - if (code) goto _err; - - tMapDataReset(&pCommitter->dWriter.mBlock); - code = tBlockDataInit(&pCommitter->dReader.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; - code = tBlockDataInit(&pCommitter->dWriter.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; - - // .data merge while (pBlock && pRow) { - int32_t c = tBlockCmprFn(pBlock, &(SBlock){.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)}); - if (c < 0) { // disk + SBlock tBlock = {.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)}; + int32_t c = tBlockCmprFn(pBlock, &tBlock); + + if (c < 0) { code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); if (code) goto _err; - // next iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); } else { pBlock = NULL; } - } else if (c > 0) { // memory + } else if (c > 0) { code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey); if (code) goto _err; - // next pRow = tsdbTbDataIterGet(pIter); if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { pRow = NULL; } - } else { // merge + } else { int32_t nOvlp = tsdbGetNumOfRowsLessThan(pIter, pBlock->maxKey); ASSERT(nOvlp > 0); @@ -1016,7 +1010,6 @@ static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter) { code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); if (code) goto _err; - // next iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); @@ -1041,21 +1034,28 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { STbDataIter iter = {0}; TSDBROW *pRow; + tMapDataReset(&pCommitter->dWriter.mBlock); + tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, &iter); pRow = tsdbTbDataIterGet(&iter); if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { pRow = NULL; } - if (pRow == NULL) { if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) == 0) { code = tMapDataCopy(&pCommitter->dReader.mBlock, &pCommitter->dWriter.mBlock); if (code) goto _err; } - goto _exit; } + code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid); + if (code) goto _err; + code = tBlockDataInit(&pCommitter->dReader.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); + if (code) goto _err; + code = tBlockDataInit(&pCommitter->dWriter.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); + if (code) goto _err; + // commit data code = tsdbMergeCommitData(pCommitter, &iter); if (code) goto _err; From b977626a78bfafc671577c0ddf854e717adc446a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 23 Aug 2022 17:42:10 +0800 Subject: [PATCH 009/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 178 +++++------------------ 1 file changed, 34 insertions(+), 144 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 4a31efdc73..2debc7b2de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -778,160 +778,51 @@ _err: return code; } -static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { - int32_t code = 0; -#if 0 - STbData *pTbData = pIter->pTbData; - int32_t nRow = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); +static int32_t tsdbCommitLastFile(SCommitter *pCommitter, STbDataIter *pIter) { + int32_t code = 0; + STbData *pTbData = pIter->pTbData; + SBlockData *pBlockData = &pCommitter->dWriter.bDatal; + TSDBROW *pRow = tsdbTbDataIterGet(pIter); - if (pCommitter->dReader.pRowInfo && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pRowInfo) == 0) { - if (pCommitter->dReader.pRowInfo->suid) { // super table - for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { - if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break; - nRow++; - } - } else { // normal table - ASSERT(pCommitter->dReader.iRow == 0); - nRow += pCommitter->dReader.bDatal.nRow; - } - } - - if (nRow == 0) goto _exit; - - TSDBROW *pRow = tsdbTbDataIterGet(pIter); if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { pRow = NULL; } - SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && pRowInfo->uid != pTbData->uid) { - pRowInfo = NULL; - } + if (pRow == NULL) goto _exit; - while (nRow) { - SBlockData *pBlockData; - int8_t toData; - - if (nRow < pCommitter->minRow) { // to .last - toData = 0; - pBlockData = &pCommitter->dWriter.bDatal; - - // commit and reset block data schema if need - // QUESTION: Is there a case that pBlockData->nRow == 0 but need to change schema ? - if (pBlockData->suid || pBlockData->uid) { - if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { - if (pBlockData->nRow > 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } - - tBlockDataReset(pBlockData); - } - } - - // set block data schema if need - if (pBlockData->suid == 0 && pBlockData->uid == 0) { - code = - tBlockDataInit(pBlockData, pTbData->suid, pTbData->suid ? 0 : pTbData->uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; - } - - if (pBlockData->nRow + nRow > pCommitter->maxRow) { + if (pBlockData->suid || pBlockData->uid) { + if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { + if (pBlockData->nRow > 0) { code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; } - } else { // to .data - toData = 1; - pBlockData = &pCommitter->dWriter.bData; - ASSERT(pBlockData->nRow == 0); + + tBlockDataReset(pBlockData); + } + } + + if (!pBlockData->suid && !pBlockData->uid) { + code = tBlockDataInit(pBlockData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); + if (code) goto _err; + } + + while (pRow) { + code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); + if (code) goto _err; + + tsdbTbDataIterNext(pIter); + pRow = tsdbTbDataIterGet(pIter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pRow = NULL; } - while (pRow && pRowInfo) { - int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); - if (c < 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; - - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); - if (code) goto _err; - - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - } else if (c > 0) { - code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid); - if (code) goto _err; - - code = tsdbCommitterNextLastRow(pCommitter); - if (code) goto _err; - - pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && pRowInfo->uid != pTbData->uid) { - pRowInfo = NULL; - } - } else { - ASSERT(0); - } - - nRow--; - if (toData) { - if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - goto _outer_break; - } - } + if (pBlockData->nRow >= pCommitter->maxRow) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; } - - while (pRow) { - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; - - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); - if (code) goto _err; - - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - - nRow--; - if (toData) { - if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - goto _outer_break; - } - } - } - - while (pRowInfo) { - code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid); - if (code) goto _err; - - code = tsdbCommitterNextLastRow(pCommitter); - if (code) goto _err; - - pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && pRowInfo->uid != pTbData->uid) { - pRowInfo = NULL; - } - - nRow--; - if (toData) { - if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - goto _outer_break; - } - } - } - - _outer_break: - ASSERT(nRow >= 0); } _exit: @@ -939,7 +830,6 @@ _exit: _err: tsdbError("vgId:%d tsdb merge commit last failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); -#endif return code; } @@ -1061,7 +951,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { if (code) goto _err; // commit last - code = tsdbMergeCommitLast(pCommitter, &iter); + code = tsdbCommitLastFile(pCommitter, &iter); if (code) goto _err; _exit: From f696ef0a6a6c8ba18533bab8cb2bdfe4d4003d68 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 23 Aug 2022 18:12:18 +0800 Subject: [PATCH 010/102] more code --- source/dnode/vnode/src/tsdb/tsdbFS.c | 77 +++++++++++++++------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index e6a90825ab..492bb22c3b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -836,27 +836,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { pSetOld->pDataF->size = pSetNew->pDataF->size; } - // last - fSet.aLastF[0] = pSetOld->aLastF[0]; - if ((!sameDisk) || (pSetOld->aLastF[0]->commitID != pSetNew->aLastF[0]->commitID)) { - pSetOld->aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (pSetOld->aLastF[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->aLastF[0] = *pSetNew->aLastF[0]; - pSetOld->aLastF[0]->nRef = 1; - - nRef = atomic_sub_fetch_32(&fSet.aLastF[0]->nRef, 1); - if (nRef == 0) { - tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.aLastF[0], fname); - taosRemoveFile(fname); - taosMemoryFree(fSet.aLastF[0]); - } - } else { - ASSERT(pSetOld->aLastF[0]->size == pSetNew->aLastF[0]->size); - } - // sma fSet.pSmaF = pSetOld->pSmaF; if ((!sameDisk) || (pSetOld->pSmaF->commitID != pSetNew->pSmaF->commitID)) { @@ -879,6 +858,27 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { pSetOld->pSmaF->size = pSetNew->pSmaF->size; } + // last + fSet.aLastF[0] = pSetOld->aLastF[0]; + if ((!sameDisk) || (pSetOld->aLastF[0]->commitID != pSetNew->aLastF[0]->commitID)) { + pSetOld->aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (pSetOld->aLastF[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aLastF[0] = *pSetNew->aLastF[0]; + pSetOld->aLastF[0]->nRef = 1; + + nRef = atomic_sub_fetch_32(&fSet.aLastF[0]->nRef, 1); + if (nRef == 0) { + tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.aLastF[0], fname); + taosRemoveFile(fname); + taosMemoryFree(fSet.aLastF[0]); + } + } else { + ASSERT(pSetOld->aLastF[0]->size == pSetNew->aLastF[0]->size); + } + if (!sameDisk) { pSetOld->diskId = pSetNew->diskId; } @@ -902,13 +902,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { taosMemoryFree(pSetOld->pDataF); } - nRef = atomic_sub_fetch_32(&pSetOld->aLastF[0]->nRef, 1); - if (nRef == 0) { - tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aLastF[0], fname); - taosRemoveFile(fname); - taosMemoryFree(pSetOld->aLastF[0]); - } - nRef = atomic_sub_fetch_32(&pSetOld->pSmaF->nRef, 1); if (nRef == 0) { tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pSmaF, fname); @@ -916,6 +909,15 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { taosMemoryFree(pSetOld->pSmaF); } + for (int8_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { + nRef = atomic_sub_fetch_32(&pSetOld->aLastF[iLast]->nRef, 1); + if (nRef == 0) { + tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aLastF[iLast], fname); + taosRemoveFile(fname); + taosMemoryFree(pSetOld->aLastF[iLast]); + } + } + taosArrayRemove(pTsdb->fs.aDFileSet, iOld); continue; @@ -942,15 +944,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { *fSet.pDataF = *pSetNew->pDataF; fSet.pDataF->nRef = 1; - // last - fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.aLastF[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *fSet.aLastF[0] = *pSetNew->aLastF[0]; - fSet.aLastF[0]->nRef = 1; - // sma fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); if (fSet.pSmaF == NULL) { @@ -960,6 +953,16 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { *fSet.pSmaF = *pSetNew->pSmaF; fSet.pSmaF->nRef = 1; + // last + ASSERT(pSetNew->nLastF == 1); + fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (fSet.aLastF[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *fSet.aLastF[0] = *pSetNew->aLastF[0]; + fSet.aLastF[0]->nRef = 1; + if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; From 1c182684c4fae38dddad2536f2cb5386800b4122 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 11:51:34 +0800 Subject: [PATCH 011/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 48 ------------------------ 1 file changed, 48 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 2debc7b2de..ea398461ae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -327,54 +327,6 @@ _exit: return code; } -// static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { -// int32_t code = 0; - -// ASSERT(pCommitter->dReader.pReader); -// ASSERT(pCommitter->dReader.pRowInfo); - -// SBlockData *pBlockDatal = &pCommitter->dReader.bDatal; -// pCommitter->dReader.iRow++; -// if (pCommitter->dReader.iRow < pBlockDatal->nRow) { -// if (pBlockDatal->uid) { -// pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; -// } else { -// pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[pCommitter->dReader.iRow]; -// } -// pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); -// } else { -// pCommitter->dReader.iBlockL++; -// if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { -// SBlockL *pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); -// int64_t suid = pBlockL->suid; -// int64_t uid = pBlockL->maxUid; - -// code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid); -// if (code) goto _exit; - -// code = tBlockDataInit(pBlockDatal, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); -// if (code) goto _exit; - -// code = tsdbReadLastBlock(pCommitter->dReader.pReader, pBlockL, pBlockDatal); -// if (code) goto _exit; - -// pCommitter->dReader.iRow = 0; -// pCommitter->dReader.pRowInfo->suid = pBlockDatal->suid; -// if (pBlockDatal->uid) { -// pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; -// } else { -// pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[0]; -// } -// pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); -// } else { -// pCommitter->dReader.pRowInfo = NULL; -// } -// } - -// _exit: -// return code; -// } - static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { int32_t code = 0; From a4d4b1040487372346a2ba0f71955eca642ca3d0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 11:58:10 +0800 Subject: [PATCH 012/102] make it compile --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 828fed8549..7d270e08ca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2316,7 +2316,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return code; } - code = tsdbReadBlockL(pReader->pFileReader, pLastBlocks); + code = tsdbReadBlockL(pReader->pFileReader, 0, pLastBlocks); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; From aa08b7647900812bd1e86f17b712bc55d76e9808 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 14:05:20 +0800 Subject: [PATCH 013/102] more work --- source/dnode/vnode/src/tsdb/tsdbFS.c | 66 +++++++++++++++++++++------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 492bb22c3b..6ebcb43ffe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -859,24 +859,60 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { } // last - fSet.aLastF[0] = pSetOld->aLastF[0]; - if ((!sameDisk) || (pSetOld->aLastF[0]->commitID != pSetNew->aLastF[0]->commitID)) { - pSetOld->aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (pSetOld->aLastF[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->aLastF[0] = *pSetNew->aLastF[0]; - pSetOld->aLastF[0]->nRef = 1; + if (sameDisk) { + if (pSetNew->nLastF > pSetOld->nLastF) { + ASSERT(pSetNew->nLastF = pSetOld->nLastF + 1); + pSetOld->aLastF[pSetOld->nLastF] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (pSetOld->aLastF[pSetOld->nLastF] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aLastF[pSetOld->nLastF] = *pSetNew->aLastF[pSetOld->nLastF]; + pSetOld->aLastF[pSetOld->nLastF]->nRef = 1; + pSetOld->nLastF++; + } else if (pSetNew->nLastF < pSetOld->nLastF) { + ASSERT(pSetNew->nLastF == 1); + for (int32_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { + SLastFile *pLastFile = pSetOld->aLastF[iLast]; + nRef = atomic_sub_fetch_32(&pLastFile->nRef, 1); + if (nRef == 0) { + tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pLastFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pLastFile); + } + pSetOld->aLastF[iLast] = NULL; + } - nRef = atomic_sub_fetch_32(&fSet.aLastF[0]->nRef, 1); - if (nRef == 0) { - tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.aLastF[0], fname); - taosRemoveFile(fname); - taosMemoryFree(fSet.aLastF[0]); + pSetOld->nLastF = 1; + pSetOld->aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (pSetOld->aLastF[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aLastF[0] = *pSetNew->aLastF[0]; + pSetOld->aLastF[0]->nRef = 1; + } else { + ASSERT(0); } } else { - ASSERT(pSetOld->aLastF[0]->size == pSetNew->aLastF[0]->size); + ASSERT(pSetOld->nLastF == pSetNew->nLastF); + for (int32_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { + SLastFile *pLastFile = pSetOld->aLastF[iLast]; + nRef = atomic_sub_fetch_32(&pLastFile->nRef, 1); + if (nRef == 0) { + tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pLastFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pLastFile); + } + + pSetOld->aLastF[iLast] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (pSetOld->aLastF[iLast] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aLastF[iLast] = *pSetNew->aLastF[iLast]; + pSetOld->aLastF[iLast]->nRef = 1; + } } if (!sameDisk) { From e39a08f76322b721b540efc656ec0496c12d612f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 14:26:53 +0800 Subject: [PATCH 014/102] fix some bug --- source/dnode/vnode/src/tsdb/tsdbFS.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 6ebcb43ffe..8db3bcf760 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -570,11 +570,6 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { } *fSet.pSmaF = *pSet->pSmaF; - if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - // last for (fSet.nLastF = 0; fSet.nLastF < pSet->nLastF; fSet.nLastF++) { fSet.aLastF[fSet.nLastF] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); @@ -584,6 +579,11 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { } *fSet.aLastF[fSet.nLastF] = *pSet->aLastF[fSet.nLastF]; } + + if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } } _exit: @@ -958,9 +958,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { continue; _add_new: - fSet.diskId = pSetNew->diskId; - fSet.fid = pSetNew->fid; - fSet.nLastF = 1; + fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nLastF = 1}; // head fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); From 5bce5c69353597402a1a6937e5c98da5ebe2111f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 14:59:55 +0800 Subject: [PATCH 015/102] more code --- source/dnode/vnode/src/inc/tsdb.h | 2 ++ source/dnode/vnode/src/tsdb/tsdbCommit.c | 11 +++++++++ source/dnode/vnode/src/tsdb/tsdbMerge.c | 29 ++++++++++++++++++++++-- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c07bf2d6c8..083bb6623f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -281,6 +281,8 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap); void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap); +// tsdbMerge.c ============================================================================================== +int32_t tsdbMerge(STsdb *pTsdb); #define TSDB_CACHE_NO(c) ((c).cacheLast == 0) #define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index ea398461ae..1fbdbdad82 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -28,6 +28,7 @@ typedef struct { typedef struct { STsdb *pTsdb; + int8_t toMerge; /* commit data */ int64_t commitID; int32_t minutes; @@ -394,6 +395,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { SDFileSet wSet = {0}; if (pRSet) { ASSERT(pRSet->nLastF < pCommitter->maxLast); + fHead = (SHeadFile){.commitID = pCommitter->commitID}; fData = *pRSet->pDataF; fSma = *pRSet->pSmaF; @@ -409,6 +411,10 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { } wSet.nLastF = pRSet->nLastF + 1; wSet.aLastF[wSet.nLastF - 1] = &fLast; // todo + + if (wSet.nLastF == pCommitter->maxLast) { + pCommitter->toMerge = 1; + } } else { fHead = (SHeadFile){.commitID = pCommitter->commitID}; fData = (SDataFile){.commitID = pCommitter->commitID}; @@ -1277,6 +1283,11 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { tsdbFSDestroy(&pCommitter->fs); taosArrayDestroy(pCommitter->aTbDataP); + if (pCommitter->toMerge) { + code = tsdbMerge(pTsdb); + if (code) goto _err; + } + tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 8b2685e97b..9bdc0ba46a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -17,11 +17,36 @@ typedef struct { STsdb *pTsdb; + int8_t maxLast; STsdbFS fs; + struct { + SDataFReader *pReader; + } dReader; + struct { + SDataFWriter *pWriter; + } dWriter; } STsdbMerger; int32_t tsdbMerge(STsdb *pTsdb) { - int32_t code = 0; - // TODO + int32_t code = 0; + STsdbMerger merger = {0}; + STsdbMerger *pMerger = &merger; + + pMerger->pTsdb = pTsdb; + pMerger->maxLast = TSDB_DEFAULT_LAST_FILE; + code = tsdbFSCopy(pTsdb, &pMerger->fs); + if (code) goto _err; + + for (int32_t iSet = 0; iSet < taosArrayGetSize(pMerger->fs.aDFileSet); iSet++) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(pMerger->fs.aDFileSet, iSet); + if (pSet->nLastF < pMerger->maxLast) continue; + + // do merge the file + } + + return code; + +_err: + tsdbError("vgId:%d tsdb merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } \ No newline at end of file From 602a48abf25cb0b5d89194a08c9bf36f7cb022ce Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 16:21:16 +0800 Subject: [PATCH 016/102] more code --- source/dnode/vnode/src/tsdb/tsdbMerge.c | 195 ++++++++++++++++++++++-- 1 file changed, 186 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 9bdc0ba46a..bbe4fead4e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -18,31 +18,208 @@ typedef struct { STsdb *pTsdb; int8_t maxLast; + int64_t commitID; STsdbFS fs; struct { SDataFReader *pReader; + SArray *aBlockIdx; + SArray *aBlockL[TSDB_MAX_LAST_FILE]; } dReader; struct { SDataFWriter *pWriter; + SArray *aBlockIdx; + SArray *aBlockL; } dWriter; } STsdbMerger; -int32_t tsdbMerge(STsdb *pTsdb) { - int32_t code = 0; - STsdbMerger merger = {0}; - STsdbMerger *pMerger = &merger; +static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { + int32_t code = 0; + STsdb *pTsdb = pMerger->pTsdb; + + // reader + code = tsdbDataFReaderOpen(&pMerger->dReader.pReader, pTsdb, pSet); + if (code) goto _err; + + code = tsdbReadBlockIdx(pMerger->dReader.pReader, pMerger->dReader.aBlockIdx); + if (code) goto _err; + + for (int8_t iLast = 0; iLast < pSet->nLastF; iLast++) { + code = tsdbReadBlockL(pMerger->dReader.pReader, iLast, pMerger->dReader.aBlockL[iLast]); + if (code) goto _err; + } + + // writer + SHeadFile fHead = {.commitID = pMerger->commitID}; + SDataFile fData = *pSet->pDataF; + SSmaFile fSma = *pSet->pSmaF; + SLastFile fLast = {.commitID = pMerger->commitID}; + SDFileSet wSet = {.diskId = pSet->diskId, + .fid = pSet->fid, + .nLastF = 1, + .pHeadF = &fHead, + .pDataF = &fData, + .pSmaF = &fSma, + .aLastF[0] = &fLast}; + code = tsdbDataFWriterOpen(&pMerger->dWriter.pWriter, pTsdb, &wSet); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d tsdb merge file data start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbMergeFileDataEnd(STsdbMerger *pMerger) { + int32_t code = 0; + STsdb *pTsdb = pMerger->pTsdb; + + // write aBlockIdx + code = tsdbWriteBlockIdx(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockIdx); + if (code) goto _err; + + // write aBlockL + code = tsdbWriteBlockL(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockL); + if (code) goto _err; + + // update file header + code = tsdbUpdateDFileSetHeader(pMerger->dWriter.pWriter); + if (code) goto _err; + + // upsert SDFileSet + code = tsdbFSUpsertFSet(&pMerger->fs, &pMerger->dWriter.pWriter->wSet); + if (code) goto _err; + + // close and sync + code = tsdbDataFWriterClose(&pMerger->dWriter.pWriter, 1); + if (code) goto _err; + + if (pMerger->dReader.pReader) { + code = tsdbDataFReaderClose(&pMerger->dReader.pReader); + if (code) goto _err; + } + + return code; + +_err: + tsdbError("vgId:%d tsdb merge file data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { + int32_t code = 0; + STsdb *pTsdb = pMerger->pTsdb; + + // start + code = tsdbMergeFileDataStart(pMerger, pSet); + if (code) goto _err; + + // impl + while (true) { + if (1) break; + // TODO + } + + // end + code = tsdbMergeFileDataEnd(pMerger); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d tsdb merge file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbStartMerge(STsdbMerger *pMerger, STsdb *pTsdb) { + int32_t code = 0; pMerger->pTsdb = pTsdb; pMerger->maxLast = TSDB_DEFAULT_LAST_FILE; + pMerger->commitID = ++pTsdb->pVnode->state.commitID; code = tsdbFSCopy(pTsdb, &pMerger->fs); + if (code) goto _exit; + + // reader + pMerger->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pMerger->dReader.aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { + pMerger->dReader.aBlockL[iLast] = taosArrayInit(0, sizeof(SBlockL)); + if (pMerger->dReader.aBlockL[iLast] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + } + + // writer + pMerger->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pMerger->dWriter.aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pMerger->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pMerger->dWriter.aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + +_exit: + return code; +} + +static int32_t tsdbEndMerge(STsdbMerger *pMerger) { + int32_t code = 0; + STsdb *pTsdb = pMerger->pTsdb; + + code = tsdbFSCommit1(pTsdb, &pMerger->fs); if (code) goto _err; - for (int32_t iSet = 0; iSet < taosArrayGetSize(pMerger->fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pMerger->fs.aDFileSet, iSet); - if (pSet->nLastF < pMerger->maxLast) continue; - - // do merge the file + taosThreadRwlockWrlock(&pTsdb->rwLock); + code = tsdbFSCommit2(pTsdb, &pMerger->fs); + if (code) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _err; } + taosThreadRwlockUnlock(&pTsdb->rwLock); + + // writer + taosArrayDestroy(pMerger->dWriter.aBlockL); + taosArrayDestroy(pMerger->dWriter.aBlockIdx); + + // reader + for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { + taosArrayDestroy(pMerger->dReader.aBlockL[iLast]); + } + taosArrayDestroy(pMerger->dReader.aBlockIdx); + tsdbFSDestroy(&pMerger->fs); + + return code; + +_err: + tsdbError("vgId:%d, tsdb end merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbMerge(STsdb *pTsdb) { + int32_t code = 0; + STsdbMerger merger = {0}; + + code = tsdbStartMerge(&merger, pTsdb); + if (code) goto _err; + + for (int32_t iSet = 0; iSet < taosArrayGetSize(merger.fs.aDFileSet); iSet++) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(merger.fs.aDFileSet, iSet); + if (pSet->nLastF < merger.maxLast) continue; + + code = tsdbMergeFileData(&merger, pSet); + if (code) goto _err; + } + + code = tsdbEndMerge(&merger); + if (code) goto _err; return code; From 66177fb9f14612f5772bc4ae1b86c7e518c94c85 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 16:41:09 +0800 Subject: [PATCH 017/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 ------ source/dnode/vnode/src/tsdb/tsdbMerge.c | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 1fbdbdad82..daba81033d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -20,12 +20,6 @@ typedef struct { STSchema *pTSchema; } SSkmInfo; -typedef struct { - int64_t suid; - int64_t uid; - TSDBROW row; -} SRowInfo; - typedef struct { STsdb *pTsdb; int8_t toMerge; diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index bbe4fead4e..e975abd482 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -15,6 +15,19 @@ #include "tsdb.h" +typedef struct { + int64_t suid; + int64_t uid; + TSDBROW row; +} SRowInfo; + +typedef struct { + SArray *aBlockL; // SArray + int32_t iBlockL; + SBlockData bData; + int32_t iRow; +} SLDataIter; + typedef struct { STsdb *pTsdb; int8_t maxLast; @@ -29,6 +42,8 @@ typedef struct { SDataFWriter *pWriter; SArray *aBlockIdx; SArray *aBlockL; + SBlockData bData; + SBlockData bDatal; } dWriter; } STsdbMerger; From f95c0013ef20b3905ef10fda97c8afb3a309805c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 17:29:28 +0800 Subject: [PATCH 018/102] more code --- include/util/trbtree.h | 56 +++++++++++++++++++++++++ source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tsdb/tsdbMerge.c | 14 +++++++ source/util/src/trbtree.c | 52 +++++++---------------- 4 files changed, 86 insertions(+), 37 deletions(-) create mode 100644 include/util/trbtree.h diff --git a/include/util/trbtree.h b/include/util/trbtree.h new file mode 100644 index 0000000000..17739b7977 --- /dev/null +++ b/include/util/trbtree.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +typedef struct SRBTree SRBTree; +typedef struct SRBTreeNode SRBTreeNode; +typedef struct SRBTreeIter SRBTreeIter; + +typedef int32_t (*tRBTreeCmprFn)(void *, void *); + +// SRBTree +#define tRBTreeCreate(compare) \ + (SRBTree) { .cmprFn = (compare), .root = NULL, .minNode = NULL, .maxNode = NULL } + +SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew); +void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode); +SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); +SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); + +// SRBTreeIter +#define tRBTreeIterCreate(tree) \ + (SRBTreeIter) { .pTree = (tree) } + +SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter); + +struct SRBTreeNode { + enum { RED, BLACK } color; + SRBTreeNode *parent; + SRBTreeNode *left; + SRBTreeNode *right; + uint8_t payload[]; +}; + +struct SRBTree { + tRBTreeCmprFn cmprFn; + SRBTreeNode *rootNode; + SRBTreeNode *minNode; + SRBTreeNode *maxNode; +}; + +struct SRBTreeIter { + SRBTree *pTree; +}; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 39c5f3873e..2de3b2b88b 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -36,6 +36,7 @@ #include "tlosertree.h" #include "tlrucache.h" #include "tmsgcb.h" +#include "trbtree.h" #include "tref.h" #include "tskiplist.h" #include "tstream.h" diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index e975abd482..34c883296e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -28,6 +28,20 @@ typedef struct { int32_t iRow; } SLDataIter; +typedef struct { + SRBTree tMerge; +} SDataMerger; + +SRowInfo *tDataMergeNext(SDataMerger *pMerger) { + SRowInfo *pRowInfo = NULL; + + SRBTreeNode *pNode = pMerger->tMerge.minNode; + if (pNode == NULL) return NULL; + + return pRowInfo; +} + +// ================================================================================ typedef struct { STsdb *pTsdb; int8_t maxLast; diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 0970485dad..e21dcacc37 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -13,30 +13,7 @@ * along with this program. If not, see . */ -#include "os.h" - -typedef int32_t (*tRBTreeCmprFn)(void *, void *); - -typedef struct SRBTree SRBTree; -typedef struct SRBTreeNode SRBTreeNode; -typedef struct SRBTreeIter SRBTreeIter; - -struct SRBTreeNode { - enum { RED, BLACK } color; - SRBTreeNode *parent; - SRBTreeNode *left; - SRBTreeNode *right; - uint8_t payload[]; -}; - -struct SRBTree { - tRBTreeCmprFn cmprFn; - SRBTreeNode *root; -}; - -struct SRBTreeIter { - SRBTree *pTree; -}; +#include "trbtree.h" #define RBTREE_NODE_COLOR(N) ((N) ? (N)->color : BLACK) @@ -51,7 +28,7 @@ static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *pNode) { right->parent = pNode->parent; if (pNode->parent == NULL) { - pTree->root = right; + pTree->rootNode = right; } else if (pNode == pNode->parent->left) { pNode->parent->left = right; } else { @@ -72,7 +49,7 @@ static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *pNode) { left->parent = pNode->parent; if (pNode->parent == NULL) { - pTree->root = left; + pTree->rootNode = left; } else if (pNode == pNode->parent->left) { pNode->parent->left = left; } else { @@ -83,20 +60,17 @@ static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *pNode) { pNode->parent = left; } -#define tRBTreeCreate(compare) \ - (SRBTree) { .cmprFn = (compare), .root = NULL } - SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { pNew->left = NULL; pNew->right = NULL; pNew->color = RED; // insert - if (pTree->root == NULL) { + if (pTree->rootNode == NULL) { pNew->parent = NULL; - pTree->root = pNew; + pTree->rootNode = pNew; } else { - SRBTreeNode *pNode = pTree->root; + SRBTreeNode *pNode = pTree->rootNode; while (true) { ASSERT(pNode); @@ -165,12 +139,16 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { } } - pTree->root->color = BLACK; + pTree->rootNode->color = BLACK; return pNew; } -SRBTreeNode *tRBTreeDrop(SRBTree *pTree, void *pKey) { - SRBTreeNode *pNode = pTree->root; +void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode) { + // TODO +} + +SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { + SRBTreeNode *pNode = pTree->rootNode; while (pNode) { int32_t c = pTree->cmprFn(pKey, pNode->payload); @@ -185,14 +163,14 @@ SRBTreeNode *tRBTreeDrop(SRBTree *pTree, void *pKey) { } if (pNode) { - // TODO + tRBTreeDrop(pTree, pNode); } return pNode; } SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { - SRBTreeNode *pNode = pTree->root; + SRBTreeNode *pNode = pTree->rootNode; while (pNode) { int32_t c = pTree->cmprFn(pKey, pNode->payload); From 04cc96d6d9bc258948fc468e3d2c16c8cba44ecc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 19:01:27 +0800 Subject: [PATCH 019/102] more code --- include/util/trbtree.h | 10 +++-- source/util/src/trbtree.c | 80 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 7 deletions(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 17739b7977..f02b746c34 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -21,7 +21,7 @@ typedef struct SRBTreeIter SRBTreeIter; typedef int32_t (*tRBTreeCmprFn)(void *, void *); -// SRBTree +// SRBTree ============================================= #define tRBTreeCreate(compare) \ (SRBTree) { .cmprFn = (compare), .root = NULL, .minNode = NULL, .maxNode = NULL } @@ -30,12 +30,13 @@ void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode); SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); -// SRBTreeIter +// SRBTreeIter ============================================= #define tRBTreeIterCreate(tree) \ - (SRBTreeIter) { .pTree = (tree) } + (SRBTreeIter) { .pTree = (tree), .pNode = (tree)->minNode } SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter); +// STRUCT ============================================= struct SRBTreeNode { enum { RED, BLACK } color; SRBTreeNode *parent; @@ -52,5 +53,6 @@ struct SRBTree { }; struct SRBTreeIter { - SRBTree *pTree; + SRBTree *pTree; + SRBTreeNode *pNode; }; diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index e21dcacc37..5b8f6c9d96 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -17,7 +17,7 @@ #define RBTREE_NODE_COLOR(N) ((N) ? (N)->color : BLACK) -// APIs ================================================ +// SRBTree ================================================ static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *pNode) { SRBTreeNode *right = pNode->right; @@ -138,13 +138,73 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { } } } - pTree->rootNode->color = BLACK; + + // update min/max node + if (pTree->minNode == NULL || pTree->cmprFn(pTree->minNode->payload, pNew->payload) > 0) { + pTree->minNode = pNew; + } + if (pTree->maxNode == NULL || pTree->cmprFn(pTree->maxNode->payload, pNew->payload) < 0) { + pTree->maxNode = pNew; + } + return pNew; } void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode) { - // TODO + // update min/max node + if (pTree->minNode == pNode) pTree->minNode = pNode->parent; + if (pTree->maxNode == pNode) pTree->maxNode = pNode->parent; + + // drop impl + if (pNode->left == NULL) { + if (pNode->parent) { + if (pNode == pNode->parent->left) { + pNode->parent->left = pNode->right; + } else { + pNode->parent->right = pNode->right; + } + } else { + pTree->rootNode = pNode->right; + } + + if (pNode->right) { + pNode->right->parent = pNode->parent; + } + } else if (pNode->right == NULL) { + if (pNode->parent) { + if (pNode == pNode->parent->left) { + pNode->parent->left = pNode->left; + } else { + pNode->parent->right = pNode->left; + } + } else { + pTree->rootNode = pNode->left; + } + + if (pNode->left) { + pNode->left->parent = pNode->parent; + } + } else { + // TODO + SRBTreeNode *pSuccessorNode = pNode->right; + while (pSuccessorNode->left) { + pSuccessorNode = pSuccessorNode->left; + } + + pSuccessorNode->parent->left = NULL; // todo: not correct here + + pSuccessorNode->parent = pNode->parent; + pSuccessorNode->left = pNode->left; + pSuccessorNode->right = pNode->right; + pNode->left->parent = pSuccessorNode; + pNode->right->parent = pSuccessorNode; + } + + // fix + if (pNode->color == BLACK) { + // TODO + } } SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { @@ -186,3 +246,17 @@ SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { return pNode; } + +// SRBTreeIter ================================================ +SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter) { + SRBTreeNode *pNode = pIter->pNode; + SRBTree *pTree = pIter->pTree; + + if (pIter->pNode) { + ASSERT(0); + // TODO + } + +_exit: + return pNode; +} \ No newline at end of file From 41f104a8686c15c01cd4d296ae40022530dd0216 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 19:11:55 +0800 Subject: [PATCH 020/102] more code --- source/util/src/trbtree.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 5b8f6c9d96..7929af96d0 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -253,8 +253,16 @@ SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter) { SRBTree *pTree = pIter->pTree; if (pIter->pNode) { - ASSERT(0); - // TODO + if (pIter->pNode->right) { + pIter->pNode = pIter->pNode->right; + while (pIter->pNode->left) { + pIter->pNode->left; + } + } else { + while (true) { + ASSERT(0); + } + } } _exit: From 0bb11b08ef806e88df037aec978c1d2c3786fb06 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 21:58:01 +0800 Subject: [PATCH 021/102] more code --- source/util/src/trbtree.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 7929af96d0..887a824151 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -260,7 +260,17 @@ SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter) { } } else { while (true) { - ASSERT(0); + if (pIter->pNode->parent) { + if (pIter->pNode == pIter->pNode->parent->left) { + pIter->pNode = pIter->pNode->parent; + break; + } else { + pIter->pNode = pIter->pNode->parent; + } + } else { + pIter->pNode = NULL; + break; + } } } } From a01ea0bed2c7e10d11cc0b95f5b76b1b4eaa5199 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 Aug 2022 22:34:11 +0800 Subject: [PATCH 022/102] more code --- include/util/trbtree.h | 17 +++++++++++++-- source/util/src/trbtree.c | 2 +- source/util/test/CMakeLists.txt | 8 +++++++ source/util/test/trbtreeTest.cpp | 36 ++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 source/util/test/trbtreeTest.cpp diff --git a/include/util/trbtree.h b/include/util/trbtree.h index f02b746c34..d6a237fb77 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -13,17 +13,24 @@ * along with this program. If not, see . */ +#ifndef _TD_UTIL_RBTREE_H_ +#define _TD_UTIL_RBTREE_H_ + #include "os.h" +#ifdef __cplusplus +extern "C" { +#endif + typedef struct SRBTree SRBTree; typedef struct SRBTreeNode SRBTreeNode; typedef struct SRBTreeIter SRBTreeIter; -typedef int32_t (*tRBTreeCmprFn)(void *, void *); +typedef int32_t (*tRBTreeCmprFn)(const void *, const void *); // SRBTree ============================================= #define tRBTreeCreate(compare) \ - (SRBTree) { .cmprFn = (compare), .root = NULL, .minNode = NULL, .maxNode = NULL } + (SRBTree) { .cmprFn = (compare), .rootNode = NULL, .minNode = NULL, .maxNode = NULL } SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew); void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode); @@ -56,3 +63,9 @@ struct SRBTreeIter { SRBTree *pTree; SRBTreeNode *pNode; }; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_RBTREE_H_*/ \ No newline at end of file diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 887a824151..3c3bad7340 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -256,7 +256,7 @@ SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter) { if (pIter->pNode->right) { pIter->pNode = pIter->pNode->right; while (pIter->pNode->left) { - pIter->pNode->left; + pIter->pNode = pIter->pNode->left; } } else { while (true) { diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index d2a503e661..6e42ef7e75 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -75,4 +75,12 @@ target_link_libraries(taosbsearchTest os util gtest_main) add_test( NAME taosbsearchTest COMMAND taosbsearchTest +) + +# trbtreeTest +add_executable(rbtreeTest "trbtreeTest.cpp") +target_link_libraries(rbtreeTest os util gtest_main) +add_test( + NAME rbtreeTest + COMMAND rbtreeTest ) \ No newline at end of file diff --git a/source/util/test/trbtreeTest.cpp b/source/util/test/trbtreeTest.cpp new file mode 100644 index 0000000000..4ff990a46d --- /dev/null +++ b/source/util/test/trbtreeTest.cpp @@ -0,0 +1,36 @@ +#include + +#include +#include + +#include "trbtree.h" + +static int32_t tCmprInteger(const void *p1, const void *p2) { + if (*(int *)p1 < *(int *)p2) { + return -1; + } else if (*(int *)p1 > *(int *)p2) { + return 1; + } + return 0; +} + +TEST(trbtreeTest, rbtree_test1) { + SRBTree rt = tRBTreeCreate(tCmprInteger); + int a[] = {1, 3, 4, 2, 7, 5, 8}; + + for (int i = 0; i < sizeof(a) / sizeof(a[0]); i++) { + SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryMalloc(sizeof(*pNode) + sizeof(int)); + *(int *)pNode->payload = a[i]; + + tRBTreePut(&rt, pNode); + } + + SRBTreeIter rti = tRBTreeIterCreate(&rt); + SRBTreeNode *pNode = tRBTreeIterNext(&rti); + int la = 0; + while (pNode) { + GTEST_ASSERT_GT(*(int *)pNode->payload, la); + la = *(int *)pNode->payload; + pNode = tRBTreeIterNext(&rti); + } +} \ No newline at end of file From b8230d5a3e16ca0512579ad68ff3a668f80daaa5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 25 Aug 2022 10:04:28 +0800 Subject: [PATCH 023/102] more code --- include/util/trbtree.h | 5 ++-- source/util/src/trbtree.c | 49 ++++++++++++++++++++++++-------- source/util/test/trbtreeTest.cpp | 2 +- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index d6a237fb77..d666cc11d1 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -38,8 +38,8 @@ SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); // SRBTreeIter ============================================= -#define tRBTreeIterCreate(tree) \ - (SRBTreeIter) { .pTree = (tree), .pNode = (tree)->minNode } +#define tRBTreeIterCreate(tree, descend) \ + (SRBTreeIter) { .des = (descend), .pTree = (tree), .pNode = (descend) ? (tree)->maxNode : (tree)->minNode } SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter); @@ -60,6 +60,7 @@ struct SRBTree { }; struct SRBTreeIter { + int8_t des; SRBTree *pTree; SRBTreeNode *pNode; }; diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 3c3bad7340..14b841dc50 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -253,23 +253,48 @@ SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter) { SRBTree *pTree = pIter->pTree; if (pIter->pNode) { - if (pIter->pNode->right) { - pIter->pNode = pIter->pNode->right; - while (pIter->pNode->left) { + if (pIter->des) { + // descend + if (pIter->pNode->left) { pIter->pNode = pIter->pNode->left; + while (pIter->pNode->right) { + pIter->pNode = pIter->pNode->right; + } + } else { + while (true) { + if (pIter->pNode->parent) { + if (pIter->pNode == pIter->pNode->parent->right) { + pIter->pNode = pIter->pNode->parent; + break; + } else { + pIter->pNode = pIter->pNode->parent; + } + } else { + pIter->pNode = NULL; + break; + } + } } } else { - while (true) { - if (pIter->pNode->parent) { - if (pIter->pNode == pIter->pNode->parent->left) { - pIter->pNode = pIter->pNode->parent; - break; + // ascend + if (pIter->pNode->right) { + pIter->pNode = pIter->pNode->right; + while (pIter->pNode->left) { + pIter->pNode = pIter->pNode->left; + } + } else { + while (true) { + if (pIter->pNode->parent) { + if (pIter->pNode == pIter->pNode->parent->left) { + pIter->pNode = pIter->pNode->parent; + break; + } else { + pIter->pNode = pIter->pNode->parent; + } } else { - pIter->pNode = pIter->pNode->parent; + pIter->pNode = NULL; + break; } - } else { - pIter->pNode = NULL; - break; } } } diff --git a/source/util/test/trbtreeTest.cpp b/source/util/test/trbtreeTest.cpp index 4ff990a46d..be62740949 100644 --- a/source/util/test/trbtreeTest.cpp +++ b/source/util/test/trbtreeTest.cpp @@ -25,7 +25,7 @@ TEST(trbtreeTest, rbtree_test1) { tRBTreePut(&rt, pNode); } - SRBTreeIter rti = tRBTreeIterCreate(&rt); + SRBTreeIter rti = tRBTreeIterCreate(&rt, 0); SRBTreeNode *pNode = tRBTreeIterNext(&rti); int la = 0; while (pNode) { From c3998092f7d4da0ee07967a4a7d3fc198677367c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 25 Aug 2022 10:43:13 +0800 Subject: [PATCH 024/102] more code --- include/util/trbtree.h | 6 +- source/util/src/trbtree.c | 120 ++++++++++++++++--------------- source/util/test/trbtreeTest.cpp | 3 +- 3 files changed, 66 insertions(+), 63 deletions(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index d666cc11d1..1e34839cbe 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -38,8 +38,8 @@ SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); // SRBTreeIter ============================================= -#define tRBTreeIterCreate(tree, descend) \ - (SRBTreeIter) { .des = (descend), .pTree = (tree), .pNode = (descend) ? (tree)->maxNode : (tree)->minNode } +#define tRBTreeIterCreate(tree, ascend) \ + (SRBTreeIter) { .asc = (ascend), .pTree = (tree), .pNode = (ascend) ? (tree)->minNode : (tree)->maxNode } SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter); @@ -60,7 +60,7 @@ struct SRBTree { }; struct SRBTreeIter { - int8_t des; + int8_t asc; SRBTree *pTree; SRBTreeNode *pNode; }; diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 14b841dc50..7364c129bb 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -60,6 +60,55 @@ static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *pNode) { pNode->parent = left; } +static SRBTreeNode *tRBTreeSuccessor(SRBTreeNode *pNode) { + if (pNode->right) { + pNode = pNode->right; + while (pNode->left) { + pNode = pNode->left; + } + } else { + while (true) { + if (pNode->parent) { + if (pNode == pNode->parent->left) { + pNode = pNode->parent; + break; + } else { + pNode = pNode->parent; + } + } else { + pNode = NULL; + break; + } + } + } + + return pNode; +} + +static SRBTreeNode *tRBTreePredecessor(SRBTreeNode *pNode) { + if (pNode->left) { + pNode = pNode->left; + while (pNode->right) { + pNode = pNode->right; + } + } else { + while (true) { + if (pNode->parent) { + if (pNode == pNode->parent->right) { + pNode = pNode->parent; + break; + } else { + pNode = pNode->parent; + } + } else { + pNode = NULL; + break; + } + } + } + return NULL; +} + SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { pNew->left = NULL; pNew->right = NULL; @@ -153,8 +202,12 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode) { // update min/max node - if (pTree->minNode == pNode) pTree->minNode = pNode->parent; - if (pTree->maxNode == pNode) pTree->maxNode = pNode->parent; + if (pTree->minNode == pNode) { + pTree->minNode = tRBTreeSuccessor(pTree->minNode); + } + if (pTree->maxNode == pNode) { + pTree->maxNode = tRBTreePredecessor(pTree->maxNode); + } // drop impl if (pNode->left == NULL) { @@ -208,19 +261,7 @@ void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode) { } SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { - SRBTreeNode *pNode = pTree->rootNode; - - while (pNode) { - int32_t c = pTree->cmprFn(pKey, pNode->payload); - - if (c < 0) { - pNode = pNode->left; - } else if (c > 0) { - pNode = pNode->right; - } else { - break; - } - } + SRBTreeNode *pNode = tRBTreeGet(pTree, pKey); if (pNode) { tRBTreeDrop(pTree, pNode); @@ -250,53 +291,14 @@ SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { // SRBTreeIter ================================================ SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter) { SRBTreeNode *pNode = pIter->pNode; - SRBTree *pTree = pIter->pTree; if (pIter->pNode) { - if (pIter->des) { - // descend - if (pIter->pNode->left) { - pIter->pNode = pIter->pNode->left; - while (pIter->pNode->right) { - pIter->pNode = pIter->pNode->right; - } - } else { - while (true) { - if (pIter->pNode->parent) { - if (pIter->pNode == pIter->pNode->parent->right) { - pIter->pNode = pIter->pNode->parent; - break; - } else { - pIter->pNode = pIter->pNode->parent; - } - } else { - pIter->pNode = NULL; - break; - } - } - } - } else { + if (pIter->asc) { // ascend - if (pIter->pNode->right) { - pIter->pNode = pIter->pNode->right; - while (pIter->pNode->left) { - pIter->pNode = pIter->pNode->left; - } - } else { - while (true) { - if (pIter->pNode->parent) { - if (pIter->pNode == pIter->pNode->parent->left) { - pIter->pNode = pIter->pNode->parent; - break; - } else { - pIter->pNode = pIter->pNode->parent; - } - } else { - pIter->pNode = NULL; - break; - } - } - } + pIter->pNode = tRBTreeSuccessor(pIter->pNode); + } else { + // descend + pIter->pNode = tRBTreePredecessor(pIter->pNode); } } diff --git a/source/util/test/trbtreeTest.cpp b/source/util/test/trbtreeTest.cpp index be62740949..1e0768c32f 100644 --- a/source/util/test/trbtreeTest.cpp +++ b/source/util/test/trbtreeTest.cpp @@ -25,12 +25,13 @@ TEST(trbtreeTest, rbtree_test1) { tRBTreePut(&rt, pNode); } - SRBTreeIter rti = tRBTreeIterCreate(&rt, 0); + SRBTreeIter rti = tRBTreeIterCreate(&rt, 1); SRBTreeNode *pNode = tRBTreeIterNext(&rti); int la = 0; while (pNode) { GTEST_ASSERT_GT(*(int *)pNode->payload, la); la = *(int *)pNode->payload; + // printf("%d\n", la); pNode = tRBTreeIterNext(&rti); } } \ No newline at end of file From beab5f1fe31053f5eff972f1fe55f2c8a82f56b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 25 Aug 2022 11:55:43 +0800 Subject: [PATCH 025/102] more code --- source/dnode/vnode/src/tsdb/tsdbMerge.c | 22 ++++++++++++- source/util/src/trbtree.c | 43 ------------------------- 2 files changed, 21 insertions(+), 44 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 34c883296e..42e9c9bc57 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -22,6 +22,7 @@ typedef struct { } SRowInfo; typedef struct { + SRowInfo rowInfo; SArray *aBlockL; // SArray int32_t iBlockL; SBlockData bData; @@ -32,7 +33,26 @@ typedef struct { SRBTree tMerge; } SDataMerger; -SRowInfo *tDataMergeNext(SDataMerger *pMerger) { +static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { + SRowInfo *pInfo1 = (SRowInfo *)p1; + SRowInfo *pInfo2 = (SRowInfo *)p2; + + if (pInfo1->suid < pInfo2->suid) { + return -1; + } else if (pInfo1->suid > pInfo2->suid) { + return 1; + } + + if (pInfo1->uid < pInfo2->uid) { + return -1; + } else if (pInfo1->uid > pInfo2->uid) { + return 1; + } + + return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row); +} + +static SRowInfo *tDataMergeNext(SDataMerger *pMerger) { SRowInfo *pRowInfo = NULL; SRBTreeNode *pNode = pMerger->tMerge.minNode; diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 7364c129bb..4b0f286ef3 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -210,49 +210,6 @@ void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode) { } // drop impl - if (pNode->left == NULL) { - if (pNode->parent) { - if (pNode == pNode->parent->left) { - pNode->parent->left = pNode->right; - } else { - pNode->parent->right = pNode->right; - } - } else { - pTree->rootNode = pNode->right; - } - - if (pNode->right) { - pNode->right->parent = pNode->parent; - } - } else if (pNode->right == NULL) { - if (pNode->parent) { - if (pNode == pNode->parent->left) { - pNode->parent->left = pNode->left; - } else { - pNode->parent->right = pNode->left; - } - } else { - pTree->rootNode = pNode->left; - } - - if (pNode->left) { - pNode->left->parent = pNode->parent; - } - } else { - // TODO - SRBTreeNode *pSuccessorNode = pNode->right; - while (pSuccessorNode->left) { - pSuccessorNode = pSuccessorNode->left; - } - - pSuccessorNode->parent->left = NULL; // todo: not correct here - - pSuccessorNode->parent = pNode->parent; - pSuccessorNode->left = pNode->left; - pSuccessorNode->right = pNode->right; - pNode->left->parent = pSuccessorNode; - pNode->right->parent = pSuccessorNode; - } // fix if (pNode->color == BLACK) { From c3304c48bf3bc1492703799b4443a965bb159447 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 25 Aug 2022 14:43:54 +0800 Subject: [PATCH 026/102] more code --- source/dnode/vnode/src/tsdb/tsdbMerge.c | 59 ++++++++++++++++++++++--- source/util/src/trbtree.c | 30 +++++++++++++ 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 42e9c9bc57..1c058fd8f1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -30,7 +30,8 @@ typedef struct { } SLDataIter; typedef struct { - SRBTree tMerge; + SRBTreeNode *pNode; + SRBTree rbt; } SDataMerger; static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { @@ -52,13 +53,59 @@ static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row); } -static SRowInfo *tDataMergeNext(SDataMerger *pMerger) { - SRowInfo *pRowInfo = NULL; +static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { + int32_t code = 0; - SRBTreeNode *pNode = pMerger->tMerge.minNode; - if (pNode == NULL) return NULL; + if (pMerger->pNode) { + // next current iter + SLDataIter *pIter = (SLDataIter *)pMerger->pNode->payload; - return pRowInfo; + pIter->iRow++; + if (pIter->iRow < pIter->bData.nRow) { + pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; + pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + } else { + pIter->iBlockL++; + if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { + code = tsdbReadLastBlock(NULL, (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL), &pIter->bData); + if (code) goto _exit; + + pIter->iRow = 0; + pIter->rowInfo.suid = pIter->bData.suid; + pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; + pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); + } else { + pMerger->pNode = NULL; + } + } + + if (pMerger->pNode && pMerger->rbt.minNode) { + int32_t c = tRowInfoCmprFn(pMerger->pNode->payload, pMerger->rbt.minNode->payload); + if (c > 0) { + pMerger->pNode = tRBTreePut(&pMerger->rbt, pMerger->pNode); + ASSERT(pMerger->pNode); + pMerger->pNode = NULL; + } else { + ASSERT(c); + } + } + } + + if (pMerger->pNode == NULL) { + pMerger->pNode = pMerger->rbt.minNode; + if (pMerger->pNode) { + tRBTreeDrop(&pMerger->rbt, pMerger->pNode); + } + } + + if (pMerger->pNode) { + *ppInfo = &((SLDataIter *)pMerger->pNode->payload)[0].rowInfo; + } else { + *ppInfo = NULL; + } + +_exit: + return code; } // ================================================================================ diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 4b0f286ef3..ba1469dce7 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -210,6 +210,36 @@ void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode) { } // drop impl + if (pNode->left == NULL) { + // transplant right + if (pNode->parent == NULL) { + pTree->rootNode = pNode->right; + } else if (pNode == pNode->parent->left) { + pNode->parent->left = pNode->right; + } else { + pNode->parent->right = pNode->right; + } + + if (pNode->right) { + pNode->right->parent = pNode->parent; + } + } else if (pNode->right == NULL) { + // transplant left + if (pNode->parent == NULL) { + pTree->rootNode = pNode->left; + } else if (pNode == pNode->parent->left) { + pNode->parent->left = pNode->left; + } else { + pNode->parent->right = pNode->left; + } + + if (pNode->left) { + pNode->left->parent = pNode->parent; + } + } else { + SRBTreeNode *y = tRBTreeSuccessor(pNode); + pNode->color = RBTREE_NODE_COLOR(y); + } // fix if (pNode->color == BLACK) { From c1462d726ae4ce8cb90e758256912221f39a2a15 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 25 Aug 2022 15:35:55 +0800 Subject: [PATCH 027/102] more code --- source/dnode/vnode/src/tsdb/tsdbMerge.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 1c058fd8f1..89c4a88606 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -53,6 +53,17 @@ static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row); } +static void tDataMergerInit(SDataMerger *pMerger, SArray *aNodeP) { + pMerger->pNode = NULL; + pMerger->rbt = tRBTreeCreate(tRowInfoCmprFn); + for (int32_t iNode = 0; iNode < taosArrayGetSize(aNodeP); iNode++) { + SRBTreeNode *pNode = (SRBTreeNode *)taosArrayGetP(aNodeP, iNode); + + pNode = tRBTreePut(&pMerger->rbt, pNode); + ASSERT(pNode); + } +} + static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { int32_t code = 0; @@ -67,7 +78,7 @@ static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { } else { pIter->iBlockL++; if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { - code = tsdbReadLastBlock(NULL, (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL), &pIter->bData); + // code = tsdbReadLastBlock(NULL, (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL), &pIter->bData); if (code) goto _exit; pIter->iRow = 0; From ffa58120530a2b55c3a763e141948cc674a2285d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 25 Aug 2022 16:21:13 +0800 Subject: [PATCH 028/102] more code --- include/util/trbtree.h | 3 +- source/util/src/trbtree.c | 192 +++++++++++++++++++++++++------------- 2 files changed, 127 insertions(+), 68 deletions(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 1e34839cbe..afc5f36e26 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -44,8 +44,9 @@ SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter); // STRUCT ============================================= +typedef enum { RED, BLACK } ECOLOR; struct SRBTreeNode { - enum { RED, BLACK } color; + ECOLOR color; SRBTreeNode *parent; SRBTreeNode *left; SRBTreeNode *right; diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index ba1469dce7..2628498e41 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -18,46 +18,40 @@ #define RBTREE_NODE_COLOR(N) ((N) ? (N)->color : BLACK) // SRBTree ================================================ -static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *pNode) { - SRBTreeNode *right = pNode->right; - - pNode->right = right->left; - if (pNode->right) { - pNode->right->parent = pNode; +static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *x) { + SRBTreeNode *y = x->right; + x->right = y->left; + if (y->left) { + y->left->parent = x; } - - right->parent = pNode->parent; - if (pNode->parent == NULL) { - pTree->rootNode = right; - } else if (pNode == pNode->parent->left) { - pNode->parent->left = right; + y->parent = x->parent; + if (x->parent == NULL) { + pTree->rootNode = y; + } else if (x == x->parent->left) { + x->parent->left = y; } else { - pNode->parent->right = right; + x->parent->right = y; } - - right->left = pNode; - pNode->parent = right; + y->left = x; + x->parent = y; } -static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *pNode) { - SRBTreeNode *left = pNode->left; - - pNode->left = left->right; - if (pNode->left) { - pNode->left->parent = pNode; +static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *x) { + SRBTreeNode *y = x->left; + x->left = y->right; + if (y->right) { + y->right->parent = x; } - - left->parent = pNode->parent; - if (pNode->parent == NULL) { - pTree->rootNode = left; - } else if (pNode == pNode->parent->left) { - pNode->parent->left = left; + y->parent = x->parent; + if (x->parent == NULL) { + pTree->rootNode = y; + } else if (x == x->parent->left) { + x->parent->left = y; } else { - pNode->parent->right = left; + x->parent->right = y; } - - left->right = pNode; - pNode->parent = left; + y->right = x; + x->parent = y; } static SRBTreeNode *tRBTreeSuccessor(SRBTreeNode *pNode) { @@ -200,50 +194,114 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { return pNew; } -void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode) { - // update min/max node - if (pTree->minNode == pNode) { - pTree->minNode = tRBTreeSuccessor(pTree->minNode); +static void tRBTreeTransplant(SRBTree *pTree, SRBTreeNode *u, SRBTreeNode *v) { + if (u->parent == NULL) { + pTree->rootNode = v; + } else if (u == u->parent->left) { + u->parent->left = v; + } else { + u->parent->right = v; } - if (pTree->maxNode == pNode) { - pTree->maxNode = tRBTreePredecessor(pTree->maxNode); + if (v) { + v->parent = u->parent; + } +} + +static void tRBTreeDropFixup(SRBTree *t, SRBTreeNode *x) { + while (x != t->rootNode && x->color == BLACK) { + if (x == x->parent->left) { + SRBTreeNode *w = x->parent->right; + if (RBTREE_NODE_COLOR(w) == RED) { + w->color = BLACK; + x->parent->color = RED; + tRBTreeRotateLeft(t, x->parent); + w = x->parent->right; + } + if (RBTREE_NODE_COLOR(w->left) == BLACK && RBTREE_NODE_COLOR(w->right) == BLACK) { + w->color = RED; + x = x->parent; + } else { + if (RBTREE_NODE_COLOR(w->right) == BLACK) { + w->left->color = BLACK; + w->color = RED; + tRBTreeRotateRight(t, w); + w = x->parent->right; + } + w->color = x->parent->color; + x->parent->color = BLACK; + w->right->color = BLACK; + tRBTreeRotateLeft(t, x->parent); + x = t->rootNode; + } + } else { + SRBTreeNode *w = x->parent->left; + if (RBTREE_NODE_COLOR(w) == RED) { + w->color = BLACK; + x->parent->color = RED; + tRBTreeRotateRight(t, x->parent); + w = x->parent->left; + } + if (RBTREE_NODE_COLOR(w->right) == BLACK && RBTREE_NODE_COLOR(w->left) == BLACK) { + w->color = RED; + x = x->parent; + } else { + if (RBTREE_NODE_COLOR(w->left) == BLACK) { + w->right->color = BLACK; + w->color = RED; + tRBTreeRotateLeft(t, w); + w = x->parent->left; + } + w->color = x->parent->color; + x->parent->color = BLACK; + w->left->color = BLACK; + tRBTreeRotateRight(t, x->parent); + x = t->rootNode; + } + } + } + x->color = BLACK; +} + +void tRBTreeDrop(SRBTree *t, SRBTreeNode *z) { + // update min/max node + if (t->minNode == z) { + t->minNode = tRBTreeSuccessor(t->minNode); + } + if (t->maxNode == z) { + t->maxNode = tRBTreePredecessor(t->maxNode); } // drop impl - if (pNode->left == NULL) { - // transplant right - if (pNode->parent == NULL) { - pTree->rootNode = pNode->right; - } else if (pNode == pNode->parent->left) { - pNode->parent->left = pNode->right; - } else { - pNode->parent->right = pNode->right; - } + SRBTreeNode *y = z; + SRBTreeNode *x; + ECOLOR oColor = y->color; - if (pNode->right) { - pNode->right->parent = pNode->parent; - } - } else if (pNode->right == NULL) { - // transplant left - if (pNode->parent == NULL) { - pTree->rootNode = pNode->left; - } else if (pNode == pNode->parent->left) { - pNode->parent->left = pNode->left; - } else { - pNode->parent->right = pNode->left; - } - - if (pNode->left) { - pNode->left->parent = pNode->parent; - } + if (z->left == NULL) { + x = z->right; + tRBTreeTransplant(t, z, z->right); + } else if (z->right == NULL) { + x = z->left; + tRBTreeTransplant(t, z, z->left); } else { - SRBTreeNode *y = tRBTreeSuccessor(pNode); - pNode->color = RBTREE_NODE_COLOR(y); + y = tRBTreeSuccessor(z); + oColor = y->color; + x = y->right; + if (y->parent == z) { + x->parent = z; + } else { + tRBTreeTransplant(t, y, y->right); + y->right = z->right; + y->right->parent = y; + } + tRBTreeTransplant(t, z, y); + y->left = z->left; + y->left->parent = y; + y->color = z->color; } // fix - if (pNode->color == BLACK) { - // TODO + if (oColor == BLACK) { + tRBTreeDropFixup(t, x); } } From cfa666f247cd7f14b50426040fa244207f891d02 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 25 Aug 2022 17:04:42 +0800 Subject: [PATCH 029/102] more code --- include/util/trbtree.h | 10 +++---- source/dnode/vnode/src/tsdb/tsdbMerge.c | 18 ++++++++---- source/util/src/trbtree.c | 38 ++++++++++++------------- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index afc5f36e26..9d5cc60666 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -30,7 +30,7 @@ typedef int32_t (*tRBTreeCmprFn)(const void *, const void *); // SRBTree ============================================= #define tRBTreeCreate(compare) \ - (SRBTree) { .cmprFn = (compare), .rootNode = NULL, .minNode = NULL, .maxNode = NULL } + (SRBTree) { .cmprFn = (compare), .root = NULL, .min = NULL, .max = NULL } SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew); void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode); @@ -39,7 +39,7 @@ SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); // SRBTreeIter ============================================= #define tRBTreeIterCreate(tree, ascend) \ - (SRBTreeIter) { .asc = (ascend), .pTree = (tree), .pNode = (ascend) ? (tree)->minNode : (tree)->maxNode } + (SRBTreeIter) { .asc = (ascend), .pTree = (tree), .pNode = (ascend) ? (tree)->min : (tree)->max } SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter); @@ -55,9 +55,9 @@ struct SRBTreeNode { struct SRBTree { tRBTreeCmprFn cmprFn; - SRBTreeNode *rootNode; - SRBTreeNode *minNode; - SRBTreeNode *maxNode; + SRBTreeNode *root; + SRBTreeNode *min; + SRBTreeNode *max; }; struct SRBTreeIter { diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 89c4a88606..878491ef7e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -90,8 +90,8 @@ static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { } } - if (pMerger->pNode && pMerger->rbt.minNode) { - int32_t c = tRowInfoCmprFn(pMerger->pNode->payload, pMerger->rbt.minNode->payload); + if (pMerger->pNode && pMerger->rbt.min) { + int32_t c = tRowInfoCmprFn(pMerger->pNode->payload, pMerger->rbt.min->payload); if (c > 0) { pMerger->pNode = tRBTreePut(&pMerger->rbt, pMerger->pNode); ASSERT(pMerger->pNode); @@ -103,7 +103,7 @@ static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { } if (pMerger->pNode == NULL) { - pMerger->pNode = pMerger->rbt.minNode; + pMerger->pNode = pMerger->rbt.min; if (pMerger->pNode) { tRBTreeDrop(&pMerger->rbt, pMerger->pNode); } @@ -128,6 +128,7 @@ typedef struct { struct { SDataFReader *pReader; SArray *aBlockIdx; + SDataMerger merger; SArray *aBlockL[TSDB_MAX_LAST_FILE]; } dReader; struct { @@ -222,9 +223,16 @@ static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { if (code) goto _err; // impl + SRowInfo rInfo = {.suid = INT64_MIN}; + SRowInfo *pInfo; while (true) { - if (1) break; - // TODO + code = tDataMergeNext(&pMerger->dReader.merger, &pInfo); + if (code) goto _err; + + if (pInfo == NULL) break; + + ASSERT(tRowInfoCmprFn(pInfo, &rInfo) > 0); + rInfo = *pInfo; } // end diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 2628498e41..02c5113904 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -26,7 +26,7 @@ static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *x) { } y->parent = x->parent; if (x->parent == NULL) { - pTree->rootNode = y; + pTree->root = y; } else if (x == x->parent->left) { x->parent->left = y; } else { @@ -44,7 +44,7 @@ static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *x) { } y->parent = x->parent; if (x->parent == NULL) { - pTree->rootNode = y; + pTree->root = y; } else if (x == x->parent->left) { x->parent->left = y; } else { @@ -109,11 +109,11 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { pNew->color = RED; // insert - if (pTree->rootNode == NULL) { + if (pTree->root == NULL) { pNew->parent = NULL; - pTree->rootNode = pNew; + pTree->root = pNew; } else { - SRBTreeNode *pNode = pTree->rootNode; + SRBTreeNode *pNode = pTree->root; while (true) { ASSERT(pNode); @@ -181,14 +181,14 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { } } } - pTree->rootNode->color = BLACK; + pTree->root->color = BLACK; // update min/max node - if (pTree->minNode == NULL || pTree->cmprFn(pTree->minNode->payload, pNew->payload) > 0) { - pTree->minNode = pNew; + if (pTree->min == NULL || pTree->cmprFn(pTree->min->payload, pNew->payload) > 0) { + pTree->min = pNew; } - if (pTree->maxNode == NULL || pTree->cmprFn(pTree->maxNode->payload, pNew->payload) < 0) { - pTree->maxNode = pNew; + if (pTree->max == NULL || pTree->cmprFn(pTree->max->payload, pNew->payload) < 0) { + pTree->max = pNew; } return pNew; @@ -196,7 +196,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { static void tRBTreeTransplant(SRBTree *pTree, SRBTreeNode *u, SRBTreeNode *v) { if (u->parent == NULL) { - pTree->rootNode = v; + pTree->root = v; } else if (u == u->parent->left) { u->parent->left = v; } else { @@ -208,7 +208,7 @@ static void tRBTreeTransplant(SRBTree *pTree, SRBTreeNode *u, SRBTreeNode *v) { } static void tRBTreeDropFixup(SRBTree *t, SRBTreeNode *x) { - while (x != t->rootNode && x->color == BLACK) { + while (x != t->root && x->color == BLACK) { if (x == x->parent->left) { SRBTreeNode *w = x->parent->right; if (RBTREE_NODE_COLOR(w) == RED) { @@ -231,7 +231,7 @@ static void tRBTreeDropFixup(SRBTree *t, SRBTreeNode *x) { x->parent->color = BLACK; w->right->color = BLACK; tRBTreeRotateLeft(t, x->parent); - x = t->rootNode; + x = t->root; } } else { SRBTreeNode *w = x->parent->left; @@ -255,7 +255,7 @@ static void tRBTreeDropFixup(SRBTree *t, SRBTreeNode *x) { x->parent->color = BLACK; w->left->color = BLACK; tRBTreeRotateRight(t, x->parent); - x = t->rootNode; + x = t->root; } } } @@ -264,11 +264,11 @@ static void tRBTreeDropFixup(SRBTree *t, SRBTreeNode *x) { void tRBTreeDrop(SRBTree *t, SRBTreeNode *z) { // update min/max node - if (t->minNode == z) { - t->minNode = tRBTreeSuccessor(t->minNode); + if (t->min == z) { + t->min = tRBTreeSuccessor(t->min); } - if (t->maxNode == z) { - t->maxNode = tRBTreePredecessor(t->maxNode); + if (t->max == z) { + t->max = tRBTreePredecessor(t->max); } // drop impl @@ -316,7 +316,7 @@ SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { } SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { - SRBTreeNode *pNode = pTree->rootNode; + SRBTreeNode *pNode = pTree->root; while (pNode) { int32_t c = pTree->cmprFn(pKey, pNode->payload); From eb2fb724b1f17b44e6ce59321ca09f781bf73d9a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 25 Aug 2022 18:15:20 +0800 Subject: [PATCH 030/102] more code --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbMerge.c | 60 +++++++++++++++---- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 17 +++++- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +- 6 files changed, 68 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 083bb6623f..f5d4af9381 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -266,7 +266,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMa int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData); -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData); +int32_t tsdbReadLastBlock(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData); // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 4963754eed..6b388c6146 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -503,7 +503,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { if (code) goto _err; } case SFSLASTNEXTROW_BLOCKDATA: - code = tsdbReadLastBlock(state->pDataFReader, state->pBlockL, state->pBlockDataL); + code = tsdbReadLastBlock(state->pDataFReader, 0, state->pBlockL, state->pBlockDataL); if (code) goto _err; state->nRow = state->blockDataL.nRow; diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 878491ef7e..b804ecf746 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -128,8 +128,8 @@ typedef struct { struct { SDataFReader *pReader; SArray *aBlockIdx; + SLDataIter aLDataiter[TSDB_MAX_LAST_FILE]; SDataMerger merger; - SArray *aBlockL[TSDB_MAX_LAST_FILE]; } dReader; struct { SDataFWriter *pWriter; @@ -140,6 +140,9 @@ typedef struct { } dWriter; } STsdbMerger; +extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, + SBlockData *pBlockData); // todo + static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { int32_t code = 0; STsdb *pTsdb = pMerger->pTsdb; @@ -151,9 +154,42 @@ static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { code = tsdbReadBlockIdx(pMerger->dReader.pReader, pMerger->dReader.aBlockIdx); if (code) goto _err; + pMerger->dReader.merger.pNode = NULL; + pMerger->dReader.merger.rbt = tRBTreeCreate(tRowInfoCmprFn); for (int8_t iLast = 0; iLast < pSet->nLastF; iLast++) { - code = tsdbReadBlockL(pMerger->dReader.pReader, iLast, pMerger->dReader.aBlockL[iLast]); + SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryCalloc(1, sizeof(*pNode) + sizeof(SLDataIter)); + if (pNode == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + SLDataIter *pIter = (SLDataIter *)pNode->payload; + + pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pIter->aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + code = tBlockDataCreate(&pIter->bData); if (code) goto _err; + + code = tsdbReadBlockL(pMerger->dReader.pReader, iLast, pIter->aBlockL); + if (code) goto _err; + + if (taosArrayGetSize(pIter->aBlockL) == 0) continue; + pIter->iBlockL = 0; + + SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, 0); + code = tsdbReadLastBlockEx(pMerger->dReader.pReader, iLast, pBlockL, &pIter->bData); + if (code) goto _err; + + pIter->iRow = 0; + pIter->rowInfo.suid = pIter->bData.suid; + pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; + pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); + + pNode = tRBTreePut(&pMerger->dReader.merger.rbt, pNode); + ASSERT(pNode); } // writer @@ -261,13 +297,13 @@ static int32_t tsdbStartMerge(STsdbMerger *pMerger, STsdb *pTsdb) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { - pMerger->dReader.aBlockL[iLast] = taosArrayInit(0, sizeof(SBlockL)); - if (pMerger->dReader.aBlockL[iLast] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } + // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { + // pMerger->dReader.aBlockL[iLast] = taosArrayInit(0, sizeof(SBlockL)); + // if (pMerger->dReader.aBlockL[iLast] == NULL) { + // code = TSDB_CODE_OUT_OF_MEMORY; + // goto _exit; + // } + // } // writer pMerger->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); @@ -305,9 +341,9 @@ static int32_t tsdbEndMerge(STsdbMerger *pMerger) { taosArrayDestroy(pMerger->dWriter.aBlockIdx); // reader - for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { - taosArrayDestroy(pMerger->dReader.aBlockL[iLast]); - } + // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { + // taosArrayDestroy(pMerger->dReader.aBlockL[iLast]); + // } taosArrayDestroy(pMerger->dReader.aBlockIdx); tsdbFSDestroy(&pMerger->fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7d270e08ca..326fc5e3f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2391,7 +2391,7 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STable return code; } - code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData); + code = tsdbReadLastBlock(pReader->pFileReader, 0, pBlock, &pLastBlockReader->lastBlockData); double el = (taosGetTimestampUs() - st) / 1000.0; if (code != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index e07b0e8f94..674e51c9d0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -908,7 +908,7 @@ _err: return code; } -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData) { +int32_t tsdbReadLastBlock(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData) { int32_t code = 0; code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, pBlockData); @@ -921,6 +921,21 @@ _err: return code; } +int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData) { + int32_t code = 0; + + // read + code = tsdbReadAndCheck(pReader->aLastFD[iLast], pBlockL->bInfo.offset, &pReader->aBuf[1], pBlockL->bInfo.szBlock, 0); + if (code) goto _exit; + + // decmpr + code = tDecmprBlockData(pReader->aBuf[1], pBlockL->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); + if (code) goto _exit; + +_exit: + return code; +} + // SDataFWriter ==================================================== int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 49778542e3..31c4e39b37 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1619,7 +1619,7 @@ _exit: int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]) { int32_t code = 0; - tBlockDataClear(pBlockData); + tBlockDataReset(pBlockData); int32_t n = 0; SDiskDataHdr hdr = {0}; From 2876f597f0da63ff0ad2c546930a83ba1689b708 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 25 Aug 2022 19:00:44 +0800 Subject: [PATCH 031/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 7 +++---- source/dnode/vnode/src/tsdb/tsdbMerge.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index daba81033d..03561ca193 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -405,10 +405,6 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { } wSet.nLastF = pRSet->nLastF + 1; wSet.aLastF[wSet.nLastF - 1] = &fLast; // todo - - if (wSet.nLastF == pCommitter->maxLast) { - pCommitter->toMerge = 1; - } } else { fHead = (SHeadFile){.commitID = pCommitter->commitID}; fData = (SDataFile){.commitID = pCommitter->commitID}; @@ -427,6 +423,9 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { wSet.nLastF = 1; wSet.aLastF[0] = &fLast; } + if (wSet.nLastF == pCommitter->maxLast) { + pCommitter->toMerge = 1; + } code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index b804ecf746..3a078131a1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -261,11 +261,13 @@ static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { // impl SRowInfo rInfo = {.suid = INT64_MIN}; SRowInfo *pInfo; + int64_t nRow = 0; while (true) { code = tDataMergeNext(&pMerger->dReader.merger, &pInfo); if (code) goto _err; if (pInfo == NULL) break; + nRow++; ASSERT(tRowInfoCmprFn(pInfo, &rInfo) > 0); rInfo = *pInfo; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 674e51c9d0..9c4a7aea13 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -925,11 +925,11 @@ int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBloc int32_t code = 0; // read - code = tsdbReadAndCheck(pReader->aLastFD[iLast], pBlockL->bInfo.offset, &pReader->aBuf[1], pBlockL->bInfo.szBlock, 0); + code = tsdbReadAndCheck(pReader->aLastFD[iLast], pBlockL->bInfo.offset, &pReader->aBuf[0], pBlockL->bInfo.szBlock, 0); if (code) goto _exit; // decmpr - code = tDecmprBlockData(pReader->aBuf[1], pBlockL->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); + code = tDecmprBlockData(pReader->aBuf[0], pBlockL->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); if (code) goto _exit; _exit: From 34abb837967a4ee1d1eef4ee0f5a43f0614b4ab5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 25 Aug 2022 23:07:59 +0800 Subject: [PATCH 032/102] more code --- include/util/trbtree.h | 13 +- source/dnode/vnode/src/tsdb/tsdbMerge.c | 11 +- source/util/src/trbtree.c | 403 ++++++++++++------------ source/util/test/trbtreeTest.cpp | 5 +- 4 files changed, 214 insertions(+), 218 deletions(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 9d5cc60666..060d5a7918 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -29,11 +29,12 @@ typedef struct SRBTreeIter SRBTreeIter; typedef int32_t (*tRBTreeCmprFn)(const void *, const void *); // SRBTree ============================================= -#define tRBTreeCreate(compare) \ - (SRBTree) { .cmprFn = (compare), .root = NULL, .min = NULL, .max = NULL } +#define tRBTreeMin(T) ((T)->min == ((T)->NIL) ? NULL : (T)->min) +#define tRBTreeMax(T) ((T)->max == ((T)->NIL) ? NULL : (T)->max) -SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew); -void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *pNode); +void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn); +SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z); +void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z); SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); @@ -50,7 +51,7 @@ struct SRBTreeNode { SRBTreeNode *parent; SRBTreeNode *left; SRBTreeNode *right; - uint8_t payload[]; + uint8_t payload[0]; }; struct SRBTree { @@ -58,6 +59,8 @@ struct SRBTree { SRBTreeNode *root; SRBTreeNode *min; SRBTreeNode *max; + SRBTreeNode *NIL; + SRBTreeNode NILNODE; }; struct SRBTreeIter { diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 3a078131a1..186db38461 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -55,7 +55,7 @@ static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { static void tDataMergerInit(SDataMerger *pMerger, SArray *aNodeP) { pMerger->pNode = NULL; - pMerger->rbt = tRBTreeCreate(tRowInfoCmprFn); + tRBTreeCreate(&pMerger->rbt, tRowInfoCmprFn); for (int32_t iNode = 0; iNode < taosArrayGetSize(aNodeP); iNode++) { SRBTreeNode *pNode = (SRBTreeNode *)taosArrayGetP(aNodeP, iNode); @@ -90,8 +90,9 @@ static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { } } - if (pMerger->pNode && pMerger->rbt.min) { - int32_t c = tRowInfoCmprFn(pMerger->pNode->payload, pMerger->rbt.min->payload); + SRBTreeNode *pMinNode = tRBTreeMin(&pMerger->rbt); + if (pMerger->pNode && pMinNode) { + int32_t c = tRowInfoCmprFn(pMerger->pNode->payload, pMinNode->payload); if (c > 0) { pMerger->pNode = tRBTreePut(&pMerger->rbt, pMerger->pNode); ASSERT(pMerger->pNode); @@ -103,7 +104,7 @@ static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { } if (pMerger->pNode == NULL) { - pMerger->pNode = pMerger->rbt.min; + pMerger->pNode = tRBTreeMin(&pMerger->rbt); if (pMerger->pNode) { tRBTreeDrop(&pMerger->rbt, pMerger->pNode); } @@ -155,7 +156,7 @@ static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { if (code) goto _err; pMerger->dReader.merger.pNode = NULL; - pMerger->dReader.merger.rbt = tRBTreeCreate(tRowInfoCmprFn); + tRBTreeCreate(&pMerger->dReader.merger.rbt, tRowInfoCmprFn); for (int8_t iLast = 0; iLast < pSet->nLastF; iLast++) { SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryCalloc(1, sizeof(*pNode) + sizeof(SLDataIter)); if (pNode == NULL) { diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 02c5113904..ba011de0d7 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -15,17 +15,14 @@ #include "trbtree.h" -#define RBTREE_NODE_COLOR(N) ((N) ? (N)->color : BLACK) - -// SRBTree ================================================ static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *x) { SRBTreeNode *y = x->right; x->right = y->left; - if (y->left) { + if (y->left != pTree->NIL) { y->left->parent = x; } y->parent = x->parent; - if (x->parent == NULL) { + if (x->parent == pTree->NIL) { pTree->root = y; } else if (x == x->parent->left) { x->parent->left = y; @@ -39,39 +36,142 @@ static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *x) { static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *x) { SRBTreeNode *y = x->left; x->left = y->right; - if (y->right) { + if (y->right != pTree->NIL) { y->right->parent = x; } y->parent = x->parent; - if (x->parent == NULL) { + if (x->parent == pTree->NIL) { pTree->root = y; - } else if (x == x->parent->left) { - x->parent->left = y; - } else { + } else if (x == x->parent->right) { x->parent->right = y; + } else { + x->parent->left = y; } y->right = x; x->parent = y; } -static SRBTreeNode *tRBTreeSuccessor(SRBTreeNode *pNode) { - if (pNode->right) { +static void tRBTreePutFix(SRBTree *pTree, SRBTreeNode *z) { + while (z->parent->color == RED) { + if (z->parent == z->parent->parent->left) { // z.parent is the left child + + SRBTreeNode *y = z->parent->parent->right; // uncle of z + + if (y->color == RED) { // case 1 + z->parent->color = BLACK; + y->color = BLACK; + z->parent->parent->color = RED; + z = z->parent->parent; + } else { // case2 or case3 + if (z == z->parent->right) { // case2 + z = z->parent; // marked z.parent as new z + tRBTreeRotateLeft(pTree, z); + } + // case3 + z->parent->color = BLACK; // made parent black + z->parent->parent->color = RED; // made parent red + tRBTreeRotateRight(pTree, z->parent->parent); + } + } else { // z.parent is the right child + SRBTreeNode *y = z->parent->parent->left; // uncle of z + + if (y->color == RED) { + z->parent->color = BLACK; + y->color = BLACK; + z->parent->parent->color = RED; + z = z->parent->parent; + } else { + if (z == z->parent->left) { + z = z->parent; // marked z.parent as new z + tRBTreeRotateRight(pTree, z); + } + z->parent->color = BLACK; // made parent black + z->parent->parent->color = RED; // made parent red + tRBTreeRotateLeft(pTree, z->parent->parent); + } + } + } + pTree->root->color = BLACK; +} + +static void tRBTreeTransplant(SRBTree *pTree, SRBTreeNode *u, SRBTreeNode *v) { + if (u->parent == pTree->NIL) + pTree->root = v; + else if (u == u->parent->left) + u->parent->left = v; + else + u->parent->right = v; + v->parent = u->parent; +} + +static void tRBTreeDropFix(SRBTree *pTree, SRBTreeNode *x) { + while (x != pTree->root && x->color == BLACK) { + if (x == x->parent->left) { + SRBTreeNode *w = x->parent->right; + if (w->color == RED) { + w->color = BLACK; + x->parent->color = RED; + tRBTreeRotateLeft(pTree, x->parent); + w = x->parent->right; + } + if (w->left->color == BLACK && w->right->color == BLACK) { + w->color = RED; + x = x->parent; + } else { + if (w->right->color == BLACK) { + w->left->color = BLACK; + w->color = RED; + tRBTreeRotateRight(pTree, w); + w = x->parent->right; + } + w->color = x->parent->color; + x->parent->color = BLACK; + w->right->color = BLACK; + tRBTreeRotateLeft(pTree, x->parent); + x = pTree->root; + } + } else { + SRBTreeNode *w = x->parent->left; + if (w->color == RED) { + w->color = BLACK; + x->parent->color = RED; + tRBTreeRotateRight(pTree, x->parent); + w = x->parent->left; + } + if (w->right->color == BLACK && w->left->color == BLACK) { + w->color = RED; + x = x->parent; + } else { + if (w->left->color == BLACK) { + w->right->color = BLACK; + w->color = RED; + tRBTreeRotateLeft(pTree, w); + w = x->parent->left; + } + w->color = x->parent->color; + x->parent->color = BLACK; + w->left->color = BLACK; + tRBTreeRotateRight(pTree, x->parent); + x = pTree->root; + } + } + } + x->color = BLACK; +} + +static SRBTreeNode *tRBTreeSuccessor(SRBTree *pTree, SRBTreeNode *pNode) { + if (pNode->right != pTree->NIL) { pNode = pNode->right; - while (pNode->left) { + while (pNode->left != pTree->NIL) { pNode = pNode->left; } } else { while (true) { - if (pNode->parent) { - if (pNode == pNode->parent->left) { - pNode = pNode->parent; - break; - } else { - pNode = pNode->parent; - } - } else { - pNode = NULL; + if (pNode->parent == pTree->NIL || pNode == pNode->parent->left) { + pNode = pNode->parent; break; + } else { + pNode = pNode->parent; } } } @@ -79,229 +179,120 @@ static SRBTreeNode *tRBTreeSuccessor(SRBTreeNode *pNode) { return pNode; } -static SRBTreeNode *tRBTreePredecessor(SRBTreeNode *pNode) { - if (pNode->left) { +static SRBTreeNode *tRBTreePredecessor(SRBTree *pTree, SRBTreeNode *pNode) { + if (pNode->left != pTree->NIL) { pNode = pNode->left; - while (pNode->right) { + while (pNode->right != pTree->NIL) { pNode = pNode->right; } } else { while (true) { - if (pNode->parent) { - if (pNode == pNode->parent->right) { - pNode = pNode->parent; - break; - } else { - pNode = pNode->parent; - } - } else { - pNode = NULL; + if (pNode->parent == pTree->NIL || pNode == pNode->parent->right) { + pNode = pNode->parent; break; + } else { + pNode = pNode->parent; } } } - return NULL; + + return pNode; } -SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { - pNew->left = NULL; - pNew->right = NULL; - pNew->color = RED; +void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn) { + pTree->cmprFn = cmprFn; + pTree->NIL = &pTree->NILNODE; + pTree->NIL->color = BLACK; + pTree->NIL->parent = NULL; + pTree->NIL->left = NULL; + pTree->NIL->right = NULL; + pTree->root = pTree->NIL; + pTree->min = pTree->NIL; + pTree->max = pTree->NIL; +} - // insert - if (pTree->root == NULL) { - pNew->parent = NULL; - pTree->root = pNew; - } else { - SRBTreeNode *pNode = pTree->root; - while (true) { - ASSERT(pNode); +SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { + SRBTreeNode *y = pTree->NIL; // variable for the parent of the added node + SRBTreeNode *temp = pTree->root; - int32_t c = pTree->cmprFn(pNew->payload, pNode->payload); - if (c < 0) { - if (pNode->left) { - pNode = pNode->left; - } else { - pNew->parent = pNode; - pNode->left = pNew; - break; - } - } else if (c > 0) { - if (pNode->right) { - pNode = pNode->right; - } else { - pNew->parent = pNode; - pNode->right = pNew; - break; - } - } else { - return NULL; - } - } - } + while (temp != pTree->NIL) { + y = temp; - // fix - SRBTreeNode *pNode = pNew; - while (pNode->parent && pNode->parent->color == RED) { - SRBTreeNode *p = pNode->parent; - SRBTreeNode *g = p->parent; - - if (p == g->left) { - SRBTreeNode *u = g->right; - - if (RBTREE_NODE_COLOR(u) == RED) { - p->color = BLACK; - u->color = BLACK; - g->color = RED; - pNode = g; - } else { - if (pNode == p->right) { - pNode = p; - tRBTreeRotateLeft(pTree, pNode); - } - pNode->parent->color = BLACK; - pNode->parent->parent->color = RED; - tRBTreeRotateRight(pTree, pNode->parent->parent); - } + int32_t c = pTree->cmprFn(z->payload, temp->payload); + if (c < 0) { + temp = temp->left; + } else if (c > 0) { + temp = temp->right; } else { - SRBTreeNode *u = g->left; - - if (RBTREE_NODE_COLOR(u) == RED) { - p->color = BLACK; - u->color = BLACK; - g->color = RED; - } else { - if (pNode == p->left) { - pNode = p; - tRBTreeRotateRight(pTree, pNode); - } - pNode->parent->color = BLACK; - pNode->parent->parent->color = RED; - tRBTreeRotateLeft(pTree, pNode->parent->parent); - } + return NULL; } } - pTree->root->color = BLACK; + z->parent = y; + + if (y == pTree->NIL) { + pTree->root = z; + } else if (pTree->cmprFn(z->payload, y->payload) < 0) { + y->left = z; + } else { + y->right = z; + } + + z->color = RED; + z->left = pTree->NIL; + z->right = pTree->NIL; + + tRBTreePutFix(pTree, z); // update min/max node - if (pTree->min == NULL || pTree->cmprFn(pTree->min->payload, pNew->payload) > 0) { - pTree->min = pNew; + if (pTree->min == pTree->NIL || pTree->cmprFn(pTree->min->payload, z->payload) > 0) { + pTree->min = z; } - if (pTree->max == NULL || pTree->cmprFn(pTree->max->payload, pNew->payload) < 0) { - pTree->max = pNew; + if (pTree->max == pTree->NIL || pTree->cmprFn(pTree->max->payload, z->payload) < 0) { + pTree->max = z; } - - return pNew; + return z; } -static void tRBTreeTransplant(SRBTree *pTree, SRBTreeNode *u, SRBTreeNode *v) { - if (u->parent == NULL) { - pTree->root = v; - } else if (u == u->parent->left) { - u->parent->left = v; - } else { - u->parent->right = v; - } - if (v) { - v->parent = u->parent; - } -} +void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z) { + SRBTreeNode *y = z; + SRBTreeNode *x; + ECOLOR y_orignal_color = y->color; -static void tRBTreeDropFixup(SRBTree *t, SRBTreeNode *x) { - while (x != t->root && x->color == BLACK) { - if (x == x->parent->left) { - SRBTreeNode *w = x->parent->right; - if (RBTREE_NODE_COLOR(w) == RED) { - w->color = BLACK; - x->parent->color = RED; - tRBTreeRotateLeft(t, x->parent); - w = x->parent->right; - } - if (RBTREE_NODE_COLOR(w->left) == BLACK && RBTREE_NODE_COLOR(w->right) == BLACK) { - w->color = RED; - x = x->parent; - } else { - if (RBTREE_NODE_COLOR(w->right) == BLACK) { - w->left->color = BLACK; - w->color = RED; - tRBTreeRotateRight(t, w); - w = x->parent->right; - } - w->color = x->parent->color; - x->parent->color = BLACK; - w->right->color = BLACK; - tRBTreeRotateLeft(t, x->parent); - x = t->root; - } - } else { - SRBTreeNode *w = x->parent->left; - if (RBTREE_NODE_COLOR(w) == RED) { - w->color = BLACK; - x->parent->color = RED; - tRBTreeRotateRight(t, x->parent); - w = x->parent->left; - } - if (RBTREE_NODE_COLOR(w->right) == BLACK && RBTREE_NODE_COLOR(w->left) == BLACK) { - w->color = RED; - x = x->parent; - } else { - if (RBTREE_NODE_COLOR(w->left) == BLACK) { - w->right->color = BLACK; - w->color = RED; - tRBTreeRotateLeft(t, w); - w = x->parent->left; - } - w->color = x->parent->color; - x->parent->color = BLACK; - w->left->color = BLACK; - tRBTreeRotateRight(t, x->parent); - x = t->root; - } - } - } - x->color = BLACK; -} - -void tRBTreeDrop(SRBTree *t, SRBTreeNode *z) { // update min/max node - if (t->min == z) { - t->min = tRBTreeSuccessor(t->min); + if (pTree->min == z) { + pTree->min = tRBTreeSuccessor(pTree, pTree->min); } - if (t->max == z) { - t->max = tRBTreePredecessor(t->max); + if (pTree->max == z) { + pTree->max = tRBTreePredecessor(pTree, pTree->max); } // drop impl - SRBTreeNode *y = z; - SRBTreeNode *x; - ECOLOR oColor = y->color; - - if (z->left == NULL) { + if (z->left == pTree->NIL) { x = z->right; - tRBTreeTransplant(t, z, z->right); - } else if (z->right == NULL) { + tRBTreeTransplant(pTree, z, z->right); + } else if (z->right == pTree->NIL) { x = z->left; - tRBTreeTransplant(t, z, z->left); + tRBTreeTransplant(pTree, z, z->left); } else { - y = tRBTreeSuccessor(z); - oColor = y->color; + y = tRBTreeSuccessor(pTree, z); + y_orignal_color = y->color; x = y->right; if (y->parent == z) { x->parent = z; } else { - tRBTreeTransplant(t, y, y->right); + tRBTreeTransplant(pTree, y, y->right); y->right = z->right; y->right->parent = y; } - tRBTreeTransplant(t, z, y); + tRBTreeTransplant(pTree, z, y); y->left = z->left; y->left->parent = y; y->color = z->color; } // fix - if (oColor == BLACK) { - tRBTreeDropFixup(t, x); + if (y_orignal_color == BLACK) { + tRBTreeDropFix(pTree, x); } } @@ -318,7 +309,7 @@ SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { SRBTreeNode *pNode = pTree->root; - while (pNode) { + while (pNode != pTree->NIL) { int32_t c = pTree->cmprFn(pKey, pNode->payload); if (c < 0) { @@ -330,23 +321,23 @@ SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { } } - return pNode; + return (pNode == pTree->NIL) ? NULL : pNode; } // SRBTreeIter ================================================ SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter) { SRBTreeNode *pNode = pIter->pNode; - if (pIter->pNode) { + if (pIter->pNode != pIter->pTree->NIL) { if (pIter->asc) { // ascend - pIter->pNode = tRBTreeSuccessor(pIter->pNode); + pIter->pNode = tRBTreeSuccessor(pIter->pTree, pIter->pNode); } else { // descend - pIter->pNode = tRBTreePredecessor(pIter->pNode); + pIter->pNode = tRBTreePredecessor(pIter->pTree, pIter->pNode); } } _exit: - return pNode; + return (pNode == pIter->pTree->NIL) ? NULL : pNode; } \ No newline at end of file diff --git a/source/util/test/trbtreeTest.cpp b/source/util/test/trbtreeTest.cpp index 1e0768c32f..518ffed2cf 100644 --- a/source/util/test/trbtreeTest.cpp +++ b/source/util/test/trbtreeTest.cpp @@ -15,8 +15,9 @@ static int32_t tCmprInteger(const void *p1, const void *p2) { } TEST(trbtreeTest, rbtree_test1) { - SRBTree rt = tRBTreeCreate(tCmprInteger); - int a[] = {1, 3, 4, 2, 7, 5, 8}; + SRBTree rt; + tRBTreeCreate(&rt, tCmprInteger); + int a[] = {1, 3, 4, 2, 7, 5, 8}; for (int i = 0; i < sizeof(a) / sizeof(a[0]); i++) { SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryMalloc(sizeof(*pNode) + sizeof(int)); From 4a1447b6664b4c9ca1db7d104a5b32c348a31fdc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 10:12:36 +0800 Subject: [PATCH 033/102] make it compile --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bdc9e0e982..6e04b67d65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2191,7 +2191,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return code; } - code = tsdbReadBlockL(pReader->pFileReader, pLastBlocks); + code = tsdbReadBlockL(pReader->pFileReader, 0, pLastBlocks); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; @@ -2266,7 +2266,7 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STable return code; } - code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData); + code = tsdbReadLastBlock(pReader->pFileReader, 0, pBlock, &pLastBlockReader->lastBlockData); double el = (taosGetTimestampUs() - st) / 1000.0; if (code != TSDB_CODE_SUCCESS) { From 039e4a0505ccb6835c4227534099a70c3fade9af Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 10:44:02 +0800 Subject: [PATCH 034/102] more code --- source/dnode/vnode/src/tsdb/tsdbMerge.c | 77 ++++++++++++++++++++----- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 186db38461..19d2cd45f3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -22,11 +22,13 @@ typedef struct { } SRowInfo; typedef struct { - SRowInfo rowInfo; - SArray *aBlockL; // SArray - int32_t iBlockL; - SBlockData bData; - int32_t iRow; + SRowInfo rowInfo; + SDataFReader *pReader; + int32_t iLast; + SArray *aBlockL; // SArray + int32_t iBlockL; + SBlockData bData; + int32_t iRow; } SLDataIter; typedef struct { @@ -64,6 +66,9 @@ static void tDataMergerInit(SDataMerger *pMerger, SArray *aNodeP) { } } +extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, + SBlockData *pBlockData); // todo + static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { int32_t code = 0; @@ -78,7 +83,8 @@ static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { } else { pIter->iBlockL++; if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { - // code = tsdbReadLastBlock(NULL, (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL), &pIter->bData); + SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); + code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pBlockL, &pIter->bData); if (code) goto _exit; pIter->iRow = 0; @@ -129,7 +135,7 @@ typedef struct { struct { SDataFReader *pReader; SArray *aBlockIdx; - SLDataIter aLDataiter[TSDB_MAX_LAST_FILE]; + SLDataIter *aLDataiter[TSDB_MAX_LAST_FILE]; SDataMerger merger; } dReader; struct { @@ -141,9 +147,6 @@ typedef struct { } dWriter; } STsdbMerger; -extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, - SBlockData *pBlockData); // todo - static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { int32_t code = 0; STsdb *pTsdb = pMerger->pTsdb; @@ -165,6 +168,8 @@ static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { } SLDataIter *pIter = (SLDataIter *)pNode->payload; + pIter->pReader = pMerger->dReader.pReader; + pIter->iLast = iLast; pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); if (pIter->aBlockL == NULL) { @@ -191,6 +196,8 @@ static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { pNode = tRBTreePut(&pMerger->dReader.merger.rbt, pNode); ASSERT(pNode); + + pMerger->dReader.aLDataiter[iLast] = pIter; } // writer @@ -251,6 +258,48 @@ _err: return code; } +typedef struct { + int64_t suid; + int64_t uid; + TSKEY ts; + int64_t version; +} SRInfo; + +static int32_t tRInfoCmprFn(const void *p1, const void *p2) { + SRInfo *pInfo1 = (SRInfo *)p1; + SRInfo *pInfo2 = (SRInfo *)p2; + + // suid + if (pInfo1->suid < pInfo2->suid) { + return -1; + } else if (pInfo1->suid > pInfo2->suid) { + return 1; + } + + // uid + if (pInfo1->uid < pInfo2->uid) { + return -1; + } else if (pInfo1->uid > pInfo2->uid) { + return 1; + } + + // ts + if (pInfo1->ts < pInfo2->ts) { + return -1; + } else if (pInfo1->ts > pInfo2->ts) { + return 1; + } + + // version + if (pInfo1->version < pInfo2->version) { + return -1; + } else if (pInfo1->version > pInfo2->version) { + return 1; + } + + return 0; +} + static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { int32_t code = 0; STsdb *pTsdb = pMerger->pTsdb; @@ -260,9 +309,9 @@ static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { if (code) goto _err; // impl - SRowInfo rInfo = {.suid = INT64_MIN}; SRowInfo *pInfo; int64_t nRow = 0; + SRInfo rInfo = {.suid = INT64_MIN}; while (true) { code = tDataMergeNext(&pMerger->dReader.merger, &pInfo); if (code) goto _err; @@ -270,8 +319,10 @@ static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { if (pInfo == NULL) break; nRow++; - ASSERT(tRowInfoCmprFn(pInfo, &rInfo) > 0); - rInfo = *pInfo; + SRInfo rInfoT = { + .suid = pInfo->suid, .uid = pInfo->uid, .ts = TSDBROW_TS(&pInfo->row), .version = TSDBROW_VERSION(&pInfo->row)}; + ASSERT(tRInfoCmprFn(&rInfoT, &rInfo) > 0); + rInfo = rInfoT; } // end From 2acfc81c8be9731aa659c805081ab717bd408dc0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 11:08:21 +0800 Subject: [PATCH 035/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 03561ca193..494faaf74d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -753,7 +753,7 @@ static int32_t tsdbCommitLastFile(SCommitter *pCommitter, STbDataIter *pIter) { } if (!pBlockData->suid && !pBlockData->uid) { - code = tBlockDataInit(pBlockData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); + code = tBlockDataInit(pBlockData, pTbData->suid, 0, pCommitter->skmTable.pTSchema); if (code) goto _err; } @@ -1150,7 +1150,7 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { tsdbCommitDataEnd(pCommitter); _exit: - tsdbDebug("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow); + tsdbInfo("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow); return code; _err: From 61cfacd8c2a785c226cbeaca03d9c6d51ec1c443 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 12:58:13 +0800 Subject: [PATCH 036/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 494faaf74d..9f4c405b33 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -388,7 +388,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { SLastFile fLast; SDFileSet wSet = {0}; if (pRSet) { - ASSERT(pRSet->nLastF < pCommitter->maxLast); + ASSERT(pCommitter->maxLast == 1 || pRSet->nLastF < pCommitter->maxLast); fHead = (SHeadFile){.commitID = pCommitter->commitID}; fData = *pRSet->pDataF; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 8db3bcf760..1a142fde1a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -650,7 +650,9 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { *pDFileSet->aLastF[0] = *pSet->aLastF[0]; pDFileSet->nLastF = 1; } else { - ASSERT(0); + for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { + *pDFileSet->aLastF[iLast] = *pSet->aLastF[iLast]; + } } goto _exit; @@ -892,7 +894,23 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { *pSetOld->aLastF[0] = *pSetNew->aLastF[0]; pSetOld->aLastF[0]->nRef = 1; } else { - ASSERT(0); + for (int32_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { + SLastFile *pLastFile = pSetOld->aLastF[iLast]; + nRef = atomic_sub_fetch_32(&pLastFile->nRef, 1); + if (nRef == 0) { + tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pLastFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pLastFile); + } + + pSetOld->aLastF[iLast] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); + if (pSetOld->aLastF[iLast] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aLastF[iLast] = *pSetNew->aLastF[iLast]; + pSetOld->aLastF[iLast]->nRef = 1; + } } } else { ASSERT(pSetOld->nLastF == pSetNew->nLastF); From c677d5494b3e7c635dbfb6e29ea36f70d25568fb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 13:03:12 +0800 Subject: [PATCH 037/102] make it compile --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 17c766a854..cfda151391 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2213,7 +2213,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return code; } - code = tsdbReadBlockL(pReader->pFileReader, pLastBlocks); + code = tsdbReadBlockL(pReader->pFileReader, 0, pLastBlocks); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; @@ -2288,7 +2288,7 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STable return code; } - code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData); + code = tsdbReadLastBlock(pReader->pFileReader, 0, pBlock, &pLastBlockReader->lastBlockData); double el = (taosGetTimestampUs() - st) / 1000.0; if (code != TSDB_CODE_SUCCESS) { From 341ff0262436d829067d3ee998f9c4f76e64936a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 13:33:16 +0800 Subject: [PATCH 038/102] more code --- source/dnode/vnode/CMakeLists.txt | 1 - source/dnode/vnode/src/tsdb/tsdbCommit.c | 388 ++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbMerge.c | 434 ----------------------- 3 files changed, 388 insertions(+), 435 deletions(-) delete mode 100644 source/dnode/vnode/src/tsdb/tsdbMerge.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a3ddad72fc..3489863f37 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -51,7 +51,6 @@ target_sources( "src/tsdb/tsdbRetention.c" "src/tsdb/tsdbDiskData.c" "src/tsdb/tsdbCompress.c" - "src/tsdb/tsdbMerge.c" "src/tsdb/tsdbCompact.c" # tq diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 9f4c405b33..0012bcd5e7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1288,3 +1288,391 @@ _err: tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } + +// Merger ===================================================================== +typedef struct { + int64_t suid; + int64_t uid; + TSDBROW row; +} SRowInfo; + +typedef struct { + SRowInfo rowInfo; + SDataFReader *pReader; + int32_t iLast; + SArray *aBlockL; // SArray + int32_t iBlockL; + SBlockData bData; + int32_t iRow; +} SLDataIter; + +typedef struct { + SRBTreeNode *pNode; + SRBTree rbt; +} SDataMerger; + +static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { + SRowInfo *pInfo1 = (SRowInfo *)p1; + SRowInfo *pInfo2 = (SRowInfo *)p2; + + if (pInfo1->suid < pInfo2->suid) { + return -1; + } else if (pInfo1->suid > pInfo2->suid) { + return 1; + } + + if (pInfo1->uid < pInfo2->uid) { + return -1; + } else if (pInfo1->uid > pInfo2->uid) { + return 1; + } + + return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row); +} + +static void tDataMergerInit(SDataMerger *pMerger, SArray *aNodeP) { + pMerger->pNode = NULL; + tRBTreeCreate(&pMerger->rbt, tRowInfoCmprFn); + for (int32_t iNode = 0; iNode < taosArrayGetSize(aNodeP); iNode++) { + SRBTreeNode *pNode = (SRBTreeNode *)taosArrayGetP(aNodeP, iNode); + + pNode = tRBTreePut(&pMerger->rbt, pNode); + ASSERT(pNode); + } +} + +extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, + SBlockData *pBlockData); // todo + +static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { + int32_t code = 0; + + if (pMerger->pNode) { + // next current iter + SLDataIter *pIter = (SLDataIter *)pMerger->pNode->payload; + + pIter->iRow++; + if (pIter->iRow < pIter->bData.nRow) { + pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; + pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + } else { + pIter->iBlockL++; + if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { + SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); + code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pBlockL, &pIter->bData); + if (code) goto _exit; + + pIter->iRow = 0; + pIter->rowInfo.suid = pIter->bData.suid; + pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; + pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); + } else { + pMerger->pNode = NULL; + } + } + + SRBTreeNode *pMinNode = tRBTreeMin(&pMerger->rbt); + if (pMerger->pNode && pMinNode) { + int32_t c = tRowInfoCmprFn(pMerger->pNode->payload, pMinNode->payload); + if (c > 0) { + pMerger->pNode = tRBTreePut(&pMerger->rbt, pMerger->pNode); + ASSERT(pMerger->pNode); + pMerger->pNode = NULL; + } else { + ASSERT(c); + } + } + } + + if (pMerger->pNode == NULL) { + pMerger->pNode = tRBTreeMin(&pMerger->rbt); + if (pMerger->pNode) { + tRBTreeDrop(&pMerger->rbt, pMerger->pNode); + } + } + + if (pMerger->pNode) { + *ppInfo = &((SLDataIter *)pMerger->pNode->payload)[0].rowInfo; + } else { + *ppInfo = NULL; + } + +_exit: + return code; +} + +// ================================================================================ +typedef struct { + STsdb *pTsdb; + int8_t maxLast; + int32_t minRow; + int32_t maxRow; + int8_t cmprAlg; + int64_t commitID; + STsdbFS fs; + struct { + SDataFReader *pReader; + SArray *aBlockIdx; + SLDataIter *aLDataiter[TSDB_MAX_LAST_FILE]; + SDataMerger merger; + } dReader; + struct { + SDataFWriter *pWriter; + SArray *aBlockIdx; + SArray *aBlockL; + SBlockData bData; + SBlockData bDatal; + } dWriter; +} STsdbMerger; + +static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { + int32_t code = 0; + STsdb *pTsdb = pMerger->pTsdb; + + // reader + code = tsdbDataFReaderOpen(&pMerger->dReader.pReader, pTsdb, pSet); + if (code) goto _err; + + code = tsdbReadBlockIdx(pMerger->dReader.pReader, pMerger->dReader.aBlockIdx); + if (code) goto _err; + + pMerger->dReader.merger.pNode = NULL; + tRBTreeCreate(&pMerger->dReader.merger.rbt, tRowInfoCmprFn); + for (int8_t iLast = 0; iLast < pSet->nLastF; iLast++) { + SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryCalloc(1, sizeof(*pNode) + sizeof(SLDataIter)); + if (pNode == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + SLDataIter *pIter = (SLDataIter *)pNode->payload; + pIter->pReader = pMerger->dReader.pReader; + pIter->iLast = iLast; + + pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pIter->aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + code = tBlockDataCreate(&pIter->bData); + if (code) goto _err; + + code = tsdbReadBlockL(pMerger->dReader.pReader, iLast, pIter->aBlockL); + if (code) goto _err; + + if (taosArrayGetSize(pIter->aBlockL) == 0) continue; + pIter->iBlockL = 0; + + SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, 0); + code = tsdbReadLastBlockEx(pMerger->dReader.pReader, iLast, pBlockL, &pIter->bData); + if (code) goto _err; + + pIter->iRow = 0; + pIter->rowInfo.suid = pIter->bData.suid; + pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; + pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); + + pNode = tRBTreePut(&pMerger->dReader.merger.rbt, pNode); + ASSERT(pNode); + + pMerger->dReader.aLDataiter[iLast] = pIter; + } + + // writer + SHeadFile fHead = {.commitID = pMerger->commitID}; + SDataFile fData = *pSet->pDataF; + SSmaFile fSma = *pSet->pSmaF; + SLastFile fLast = {.commitID = pMerger->commitID}; + SDFileSet wSet = {.diskId = pSet->diskId, + .fid = pSet->fid, + .nLastF = 1, + .pHeadF = &fHead, + .pDataF = &fData, + .pSmaF = &fSma, + .aLastF[0] = &fLast}; + code = tsdbDataFWriterOpen(&pMerger->dWriter.pWriter, pTsdb, &wSet); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d tsdb merge file data start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbMergeFileDataEnd(STsdbMerger *pMerger) { + int32_t code = 0; + STsdb *pTsdb = pMerger->pTsdb; + + // write aBlockIdx + code = tsdbWriteBlockIdx(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockIdx); + if (code) goto _err; + + // write aBlockL + code = tsdbWriteBlockL(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockL); + if (code) goto _err; + + // update file header + code = tsdbUpdateDFileSetHeader(pMerger->dWriter.pWriter); + if (code) goto _err; + + // upsert SDFileSet + code = tsdbFSUpsertFSet(&pMerger->fs, &pMerger->dWriter.pWriter->wSet); + if (code) goto _err; + + // close and sync + code = tsdbDataFWriterClose(&pMerger->dWriter.pWriter, 1); + if (code) goto _err; + + if (pMerger->dReader.pReader) { + code = tsdbDataFReaderClose(&pMerger->dReader.pReader); + if (code) goto _err; + } + + return code; + +_err: + tsdbError("vgId:%d tsdb merge file data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { + int32_t code = 0; + STsdb *pTsdb = pMerger->pTsdb; + + // start + code = tsdbMergeFileDataStart(pMerger, pSet); + if (code) goto _err; + + // impl + SRowInfo *pInfo; + TABLEID id = {0}; + while (true) { + code = tDataMergeNext(&pMerger->dReader.merger, &pInfo); + if (code) goto _err; + + if (pInfo == NULL) { + // end commit (todo) + break; + } + + if (id.suid != pInfo->suid || id.uid != pInfo->uid) { + // table changed, do something (todo) + } + + code = tBlockDataAppendRow(&pMerger->dWriter.bData, &pInfo->row, NULL, pInfo->uid); + if (code) goto _err; + + if (pMerger->dWriter.bData.nRow >= pMerger->maxRow * 4 / 5) { + // code = tsdbCommitDataBlock(); + if (code) goto _err; + } + } + + // end + code = tsdbMergeFileDataEnd(pMerger); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d tsdb merge file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbStartMerge(STsdbMerger *pMerger, STsdb *pTsdb) { + int32_t code = 0; + + pMerger->pTsdb = pTsdb; + pMerger->maxLast = TSDB_DEFAULT_LAST_FILE; + pMerger->commitID = ++pTsdb->pVnode->state.commitID; + code = tsdbFSCopy(pTsdb, &pMerger->fs); + if (code) goto _exit; + + // reader + pMerger->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pMerger->dReader.aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { + // pMerger->dReader.aBlockL[iLast] = taosArrayInit(0, sizeof(SBlockL)); + // if (pMerger->dReader.aBlockL[iLast] == NULL) { + // code = TSDB_CODE_OUT_OF_MEMORY; + // goto _exit; + // } + // } + + // writer + pMerger->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pMerger->dWriter.aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pMerger->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pMerger->dWriter.aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + +_exit: + return code; +} + +static int32_t tsdbEndMerge(STsdbMerger *pMerger) { + int32_t code = 0; + STsdb *pTsdb = pMerger->pTsdb; + + code = tsdbFSCommit1(pTsdb, &pMerger->fs); + if (code) goto _err; + + taosThreadRwlockWrlock(&pTsdb->rwLock); + code = tsdbFSCommit2(pTsdb, &pMerger->fs); + if (code) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _err; + } + taosThreadRwlockUnlock(&pTsdb->rwLock); + + // writer + taosArrayDestroy(pMerger->dWriter.aBlockL); + taosArrayDestroy(pMerger->dWriter.aBlockIdx); + + // reader + // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { + // taosArrayDestroy(pMerger->dReader.aBlockL[iLast]); + // } + taosArrayDestroy(pMerger->dReader.aBlockIdx); + tsdbFSDestroy(&pMerger->fs); + + return code; + +_err: + tsdbError("vgId:%d, tsdb end merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbMerge(STsdb *pTsdb) { + int32_t code = 0; + STsdbMerger merger = {0}; + + code = tsdbStartMerge(&merger, pTsdb); + if (code) goto _err; + + for (int32_t iSet = 0; iSet < taosArrayGetSize(merger.fs.aDFileSet); iSet++) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(merger.fs.aDFileSet, iSet); + if (pSet->nLastF < merger.maxLast) continue; + + code = tsdbMergeFileData(&merger, pSet); + if (code) goto _err; + } + + code = tsdbEndMerge(&merger); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d tsdb merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c deleted file mode 100644 index 19d2cd45f3..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ /dev/null @@ -1,434 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdb.h" - -typedef struct { - int64_t suid; - int64_t uid; - TSDBROW row; -} SRowInfo; - -typedef struct { - SRowInfo rowInfo; - SDataFReader *pReader; - int32_t iLast; - SArray *aBlockL; // SArray - int32_t iBlockL; - SBlockData bData; - int32_t iRow; -} SLDataIter; - -typedef struct { - SRBTreeNode *pNode; - SRBTree rbt; -} SDataMerger; - -static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { - SRowInfo *pInfo1 = (SRowInfo *)p1; - SRowInfo *pInfo2 = (SRowInfo *)p2; - - if (pInfo1->suid < pInfo2->suid) { - return -1; - } else if (pInfo1->suid > pInfo2->suid) { - return 1; - } - - if (pInfo1->uid < pInfo2->uid) { - return -1; - } else if (pInfo1->uid > pInfo2->uid) { - return 1; - } - - return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row); -} - -static void tDataMergerInit(SDataMerger *pMerger, SArray *aNodeP) { - pMerger->pNode = NULL; - tRBTreeCreate(&pMerger->rbt, tRowInfoCmprFn); - for (int32_t iNode = 0; iNode < taosArrayGetSize(aNodeP); iNode++) { - SRBTreeNode *pNode = (SRBTreeNode *)taosArrayGetP(aNodeP, iNode); - - pNode = tRBTreePut(&pMerger->rbt, pNode); - ASSERT(pNode); - } -} - -extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, - SBlockData *pBlockData); // todo - -static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { - int32_t code = 0; - - if (pMerger->pNode) { - // next current iter - SLDataIter *pIter = (SLDataIter *)pMerger->pNode->payload; - - pIter->iRow++; - if (pIter->iRow < pIter->bData.nRow) { - pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; - pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); - } else { - pIter->iBlockL++; - if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { - SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); - code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pBlockL, &pIter->bData); - if (code) goto _exit; - - pIter->iRow = 0; - pIter->rowInfo.suid = pIter->bData.suid; - pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; - pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); - } else { - pMerger->pNode = NULL; - } - } - - SRBTreeNode *pMinNode = tRBTreeMin(&pMerger->rbt); - if (pMerger->pNode && pMinNode) { - int32_t c = tRowInfoCmprFn(pMerger->pNode->payload, pMinNode->payload); - if (c > 0) { - pMerger->pNode = tRBTreePut(&pMerger->rbt, pMerger->pNode); - ASSERT(pMerger->pNode); - pMerger->pNode = NULL; - } else { - ASSERT(c); - } - } - } - - if (pMerger->pNode == NULL) { - pMerger->pNode = tRBTreeMin(&pMerger->rbt); - if (pMerger->pNode) { - tRBTreeDrop(&pMerger->rbt, pMerger->pNode); - } - } - - if (pMerger->pNode) { - *ppInfo = &((SLDataIter *)pMerger->pNode->payload)[0].rowInfo; - } else { - *ppInfo = NULL; - } - -_exit: - return code; -} - -// ================================================================================ -typedef struct { - STsdb *pTsdb; - int8_t maxLast; - int64_t commitID; - STsdbFS fs; - struct { - SDataFReader *pReader; - SArray *aBlockIdx; - SLDataIter *aLDataiter[TSDB_MAX_LAST_FILE]; - SDataMerger merger; - } dReader; - struct { - SDataFWriter *pWriter; - SArray *aBlockIdx; - SArray *aBlockL; - SBlockData bData; - SBlockData bDatal; - } dWriter; -} STsdbMerger; - -static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { - int32_t code = 0; - STsdb *pTsdb = pMerger->pTsdb; - - // reader - code = tsdbDataFReaderOpen(&pMerger->dReader.pReader, pTsdb, pSet); - if (code) goto _err; - - code = tsdbReadBlockIdx(pMerger->dReader.pReader, pMerger->dReader.aBlockIdx); - if (code) goto _err; - - pMerger->dReader.merger.pNode = NULL; - tRBTreeCreate(&pMerger->dReader.merger.rbt, tRowInfoCmprFn); - for (int8_t iLast = 0; iLast < pSet->nLastF; iLast++) { - SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryCalloc(1, sizeof(*pNode) + sizeof(SLDataIter)); - if (pNode == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - SLDataIter *pIter = (SLDataIter *)pNode->payload; - pIter->pReader = pMerger->dReader.pReader; - pIter->iLast = iLast; - - pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pIter->aBlockL == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - code = tBlockDataCreate(&pIter->bData); - if (code) goto _err; - - code = tsdbReadBlockL(pMerger->dReader.pReader, iLast, pIter->aBlockL); - if (code) goto _err; - - if (taosArrayGetSize(pIter->aBlockL) == 0) continue; - pIter->iBlockL = 0; - - SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, 0); - code = tsdbReadLastBlockEx(pMerger->dReader.pReader, iLast, pBlockL, &pIter->bData); - if (code) goto _err; - - pIter->iRow = 0; - pIter->rowInfo.suid = pIter->bData.suid; - pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; - pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); - - pNode = tRBTreePut(&pMerger->dReader.merger.rbt, pNode); - ASSERT(pNode); - - pMerger->dReader.aLDataiter[iLast] = pIter; - } - - // writer - SHeadFile fHead = {.commitID = pMerger->commitID}; - SDataFile fData = *pSet->pDataF; - SSmaFile fSma = *pSet->pSmaF; - SLastFile fLast = {.commitID = pMerger->commitID}; - SDFileSet wSet = {.diskId = pSet->diskId, - .fid = pSet->fid, - .nLastF = 1, - .pHeadF = &fHead, - .pDataF = &fData, - .pSmaF = &fSma, - .aLastF[0] = &fLast}; - code = tsdbDataFWriterOpen(&pMerger->dWriter.pWriter, pTsdb, &wSet); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d tsdb merge file data start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbMergeFileDataEnd(STsdbMerger *pMerger) { - int32_t code = 0; - STsdb *pTsdb = pMerger->pTsdb; - - // write aBlockIdx - code = tsdbWriteBlockIdx(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockIdx); - if (code) goto _err; - - // write aBlockL - code = tsdbWriteBlockL(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockL); - if (code) goto _err; - - // update file header - code = tsdbUpdateDFileSetHeader(pMerger->dWriter.pWriter); - if (code) goto _err; - - // upsert SDFileSet - code = tsdbFSUpsertFSet(&pMerger->fs, &pMerger->dWriter.pWriter->wSet); - if (code) goto _err; - - // close and sync - code = tsdbDataFWriterClose(&pMerger->dWriter.pWriter, 1); - if (code) goto _err; - - if (pMerger->dReader.pReader) { - code = tsdbDataFReaderClose(&pMerger->dReader.pReader); - if (code) goto _err; - } - - return code; - -_err: - tsdbError("vgId:%d tsdb merge file data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -typedef struct { - int64_t suid; - int64_t uid; - TSKEY ts; - int64_t version; -} SRInfo; - -static int32_t tRInfoCmprFn(const void *p1, const void *p2) { - SRInfo *pInfo1 = (SRInfo *)p1; - SRInfo *pInfo2 = (SRInfo *)p2; - - // suid - if (pInfo1->suid < pInfo2->suid) { - return -1; - } else if (pInfo1->suid > pInfo2->suid) { - return 1; - } - - // uid - if (pInfo1->uid < pInfo2->uid) { - return -1; - } else if (pInfo1->uid > pInfo2->uid) { - return 1; - } - - // ts - if (pInfo1->ts < pInfo2->ts) { - return -1; - } else if (pInfo1->ts > pInfo2->ts) { - return 1; - } - - // version - if (pInfo1->version < pInfo2->version) { - return -1; - } else if (pInfo1->version > pInfo2->version) { - return 1; - } - - return 0; -} - -static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { - int32_t code = 0; - STsdb *pTsdb = pMerger->pTsdb; - - // start - code = tsdbMergeFileDataStart(pMerger, pSet); - if (code) goto _err; - - // impl - SRowInfo *pInfo; - int64_t nRow = 0; - SRInfo rInfo = {.suid = INT64_MIN}; - while (true) { - code = tDataMergeNext(&pMerger->dReader.merger, &pInfo); - if (code) goto _err; - - if (pInfo == NULL) break; - nRow++; - - SRInfo rInfoT = { - .suid = pInfo->suid, .uid = pInfo->uid, .ts = TSDBROW_TS(&pInfo->row), .version = TSDBROW_VERSION(&pInfo->row)}; - ASSERT(tRInfoCmprFn(&rInfoT, &rInfo) > 0); - rInfo = rInfoT; - } - - // end - code = tsdbMergeFileDataEnd(pMerger); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d tsdb merge file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbStartMerge(STsdbMerger *pMerger, STsdb *pTsdb) { - int32_t code = 0; - - pMerger->pTsdb = pTsdb; - pMerger->maxLast = TSDB_DEFAULT_LAST_FILE; - pMerger->commitID = ++pTsdb->pVnode->state.commitID; - code = tsdbFSCopy(pTsdb, &pMerger->fs); - if (code) goto _exit; - - // reader - pMerger->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pMerger->dReader.aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { - // pMerger->dReader.aBlockL[iLast] = taosArrayInit(0, sizeof(SBlockL)); - // if (pMerger->dReader.aBlockL[iLast] == NULL) { - // code = TSDB_CODE_OUT_OF_MEMORY; - // goto _exit; - // } - // } - - // writer - pMerger->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pMerger->dWriter.aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - pMerger->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pMerger->dWriter.aBlockL == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - -_exit: - return code; -} - -static int32_t tsdbEndMerge(STsdbMerger *pMerger) { - int32_t code = 0; - STsdb *pTsdb = pMerger->pTsdb; - - code = tsdbFSCommit1(pTsdb, &pMerger->fs); - if (code) goto _err; - - taosThreadRwlockWrlock(&pTsdb->rwLock); - code = tsdbFSCommit2(pTsdb, &pMerger->fs); - if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; - } - taosThreadRwlockUnlock(&pTsdb->rwLock); - - // writer - taosArrayDestroy(pMerger->dWriter.aBlockL); - taosArrayDestroy(pMerger->dWriter.aBlockIdx); - - // reader - // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { - // taosArrayDestroy(pMerger->dReader.aBlockL[iLast]); - // } - taosArrayDestroy(pMerger->dReader.aBlockIdx); - tsdbFSDestroy(&pMerger->fs); - - return code; - -_err: - tsdbError("vgId:%d, tsdb end merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbMerge(STsdb *pTsdb) { - int32_t code = 0; - STsdbMerger merger = {0}; - - code = tsdbStartMerge(&merger, pTsdb); - if (code) goto _err; - - for (int32_t iSet = 0; iSet < taosArrayGetSize(merger.fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(merger.fs.aDFileSet, iSet); - if (pSet->nLastF < merger.maxLast) continue; - - code = tsdbMergeFileData(&merger, pSet); - if (code) goto _err; - } - - code = tsdbEndMerge(&merger); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d tsdb merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} \ No newline at end of file From 01fe1fc80c85e93f479542780be7e55dac17b5f1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 13:36:42 +0800 Subject: [PATCH 039/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 0012bcd5e7..26e691b249 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1552,12 +1552,23 @@ static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { if (code) goto _err; if (pInfo == NULL) { - // end commit (todo) + if (pMerger->dWriter.bData.nRow > 0) { + // TODO + } + + if (pMerger->dWriter.bDatal.nRow > 0) { + // TODO + } + break; } if (id.suid != pInfo->suid || id.uid != pInfo->uid) { - // table changed, do something (todo) + while (true) { + // move commit the head data + } + + // prepare to commit next } code = tBlockDataAppendRow(&pMerger->dWriter.bData, &pInfo->row, NULL, pInfo->uid); From 393ab4de7cfc3c20c869d548b2eb3432739a9ba3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 15:35:43 +0800 Subject: [PATCH 040/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 219 ++++++++++++++++++++--- 1 file changed, 193 insertions(+), 26 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 26e691b249..4b43871096 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -20,6 +20,32 @@ typedef struct { STSchema *pTSchema; } SSkmInfo; +typedef struct { + int64_t suid; + int64_t uid; + TSDBROW row; +} SRowInfo; + +typedef struct { + SRBTreeNode n; + SRowInfo r; + int8_t type; + union { + struct { + SArray *aTbDataP; + int32_t iTbDataP; + STbDataIter iter; + }; // memory data iter + struct { + int32_t iLast; + SArray *aBlockL; + int32_t iBlockL; + SBlockData bData; + int32_t iRow; + }; // last file data iter + }; +} SDataIter; + typedef struct { STsdb *pTsdb; int8_t toMerge; @@ -47,6 +73,10 @@ typedef struct { SMapData mBlock; // SMapData SBlockData bData; } dReader; + struct { + SDataIter *pIter; + SRBTree rbt; + }; struct { SDataFWriter *pWriter; SArray *aBlockIdx; // SArray @@ -65,6 +95,9 @@ typedef struct { SArray *aDelData; // SArray } SCommitter; +extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, + SBlockData *pBlockData); // todo + static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter); static int32_t tsdbCommitData(SCommitter *pCommitter); static int32_t tsdbCommitDel(SCommitter *pCommitter); @@ -1289,28 +1322,7 @@ _err: return code; } -// Merger ===================================================================== -typedef struct { - int64_t suid; - int64_t uid; - TSDBROW row; -} SRowInfo; - -typedef struct { - SRowInfo rowInfo; - SDataFReader *pReader; - int32_t iLast; - SArray *aBlockL; // SArray - int32_t iBlockL; - SBlockData bData; - int32_t iRow; -} SLDataIter; - -typedef struct { - SRBTreeNode *pNode; - SRBTree rbt; -} SDataMerger; - +// ================================================================================ static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { SRowInfo *pInfo1 = (SRowInfo *)p1; SRowInfo *pInfo2 = (SRowInfo *)p2; @@ -1330,6 +1342,165 @@ static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row); } +static int32_t tsdbNextCommitRow(SCommitter *pCommitter, SRowInfo **ppInfo) { + int32_t code = 0; + + if (pCommitter->pIter) { + SDataIter *pIter = pCommitter->pIter; + if (pCommitter->pIter->type == 0) { // memory + tsdbTbDataIterNext(&pIter->iter); + TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter); + while (true) { + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); + pRow = NULL; + } + + if (pRow) { + pIter->r.suid = pIter->iter.pTbData->suid; + pIter->r.uid = pIter->iter.pTbData->uid; + pIter->r.row = *pRow; + break; + } + + pIter->iTbDataP++; + if (pIter->iTbDataP < taosArrayGetSize(pIter->aTbDataP)) { + STbData *pTbData = (STbData *)taosArrayGetP(pIter->aTbDataP, pIter->iTbDataP); + TSDBKEY keyFrom = {.ts = pCommitter->minKey, .version = VERSION_MIN}; + tsdbTbDataIterOpen(pTbData, &keyFrom, 0, &pIter->iter); + pRow = tsdbTbDataIterGet(&pIter->iter); + continue; + } else { + pCommitter->pIter = NULL; + break; + } + } + } else if (pCommitter->pIter->type == 1) { // last file + pIter->iRow++; + if (pIter->iRow < pIter->bData.nRow) { + pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; + pIter->r.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + } else { + pIter->iBlockL++; + if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { + SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); + + code = tsdbReadLastBlockEx(pCommitter->dReader.pReader, pIter->iLast, pBlockL, &pIter->bData); + if (code) goto _exit; + + pIter->iRow = 0; + pIter->r.suid = pIter->bData.suid; + pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; + pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0); + } else { + pCommitter->pIter = NULL; + } + } + } else { + ASSERT(0); + } + + // compare with min in RB Tree + pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt); + if (pCommitter->pIter && pIter) { + int32_t c = tRowInfoCmprFn(&pCommitter->pIter->r, &pIter->r); + if (c > 0) { + tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter); + pCommitter->pIter = NULL; + } else { + ASSERT(c); + } + } + } + + if (pCommitter->pIter == NULL) { + pCommitter->pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt); + if (pCommitter->pIter) { + tRBTreeDrop(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter); + } + } + + if (pCommitter->pIter) { + *ppInfo = &pCommitter->pIter->r; + } else { + *ppInfo = NULL; + } + +_exit: + return code; +} + +static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { + int32_t code = 0; + + SRowInfo *pRowInfo = NULL; + TABLEID id = {0}; + while (true) { + code = tsdbNextCommitRow(pCommitter, &pRowInfo); + if (code) goto _err; + + if (pRowInfo == NULL) { + // end the commit (todo) + break; + } + + if (id.suid != pRowInfo->suid || id.uid != pRowInfo->uid) { + // table changed, end current table commit (todo) + + // prepare the new + id.suid = pRowInfo->suid; + id.uid = pRowInfo->uid; + } + + SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx; + if (pBlockIdx && pBlockIdx->suid == id.suid && pBlockIdx->uid == id.uid) { + while (true) { + /* code */ + } + } + + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, pRowInfo->suid, pRowInfo->uid, TSDBROW_SVERSION(&pRowInfo->row)); + if (code) goto _err; + } + + code = tBlockDataAppendRow(&pCommitter->dWriter.bData, &pRowInfo->row, pCommitter->skmRow.pTSchema, pRowInfo->uid); + if (code) goto _err; + + if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) { + if (1 /*toLastOnly*/) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } else { + code = tsdbCommitDataBlock(pCommitter, NULL); + if (code) goto _err; + } + } + } + + return code; + +_err: + tsdbError("vgId:%d tsdb commit file data impl failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +// ================================================================================ +typedef struct { + SRowInfo rowInfo; + SDataFReader *pReader; + int32_t iLast; + SArray *aBlockL; // SArray + int32_t iBlockL; + SBlockData bData; + int32_t iRow; +} SLDataIter; + +typedef struct { + SRBTreeNode *pNode; + SRBTree rbt; +} SDataMerger; + static void tDataMergerInit(SDataMerger *pMerger, SArray *aNodeP) { pMerger->pNode = NULL; tRBTreeCreate(&pMerger->rbt, tRowInfoCmprFn); @@ -1341,9 +1512,6 @@ static void tDataMergerInit(SDataMerger *pMerger, SArray *aNodeP) { } } -extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, - SBlockData *pBlockData); // todo - static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { int32_t code = 0; @@ -1401,7 +1569,6 @@ _exit: return code; } -// ================================================================================ typedef struct { STsdb *pTsdb; int8_t maxLast; From 90b32809c5cebc7810223d885228bf6f02b27678 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 15:48:44 +0800 Subject: [PATCH 041/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 753 ++++++++++++----------- 1 file changed, 380 insertions(+), 373 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 4b43871096..b440a61469 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1024,6 +1024,7 @@ _err: return code; } +static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter); static int32_t tsdbCommitFileData(SCommitter *pCommitter) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; @@ -1033,6 +1034,11 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { code = tsdbCommitFileDataStart(pCommitter); if (code) goto _err; +#if 1 + // impl + code = tsdbCommitFileDataImpl(pCommitter); + if (code) goto _err; +#else // commit file data impl for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pCommitter->aTbDataP); iTbData++) { STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); @@ -1059,6 +1065,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; } +#endif // commit file data end code = tsdbCommitFileDataEnd(pCommitter); @@ -1309,10 +1316,10 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { tsdbFSDestroy(&pCommitter->fs); taosArrayDestroy(pCommitter->aTbDataP); - if (pCommitter->toMerge) { - code = tsdbMerge(pTsdb); - if (code) goto _err; - } + // if (pCommitter->toMerge) { + // code = tsdbMerge(pTsdb); + // if (code) goto _err; + // } tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); return code; @@ -1485,372 +1492,372 @@ _err: return code; } -// ================================================================================ -typedef struct { - SRowInfo rowInfo; - SDataFReader *pReader; - int32_t iLast; - SArray *aBlockL; // SArray - int32_t iBlockL; - SBlockData bData; - int32_t iRow; -} SLDataIter; - -typedef struct { - SRBTreeNode *pNode; - SRBTree rbt; -} SDataMerger; - -static void tDataMergerInit(SDataMerger *pMerger, SArray *aNodeP) { - pMerger->pNode = NULL; - tRBTreeCreate(&pMerger->rbt, tRowInfoCmprFn); - for (int32_t iNode = 0; iNode < taosArrayGetSize(aNodeP); iNode++) { - SRBTreeNode *pNode = (SRBTreeNode *)taosArrayGetP(aNodeP, iNode); - - pNode = tRBTreePut(&pMerger->rbt, pNode); - ASSERT(pNode); - } -} - -static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { - int32_t code = 0; - - if (pMerger->pNode) { - // next current iter - SLDataIter *pIter = (SLDataIter *)pMerger->pNode->payload; - - pIter->iRow++; - if (pIter->iRow < pIter->bData.nRow) { - pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; - pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); - } else { - pIter->iBlockL++; - if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { - SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); - code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pBlockL, &pIter->bData); - if (code) goto _exit; - - pIter->iRow = 0; - pIter->rowInfo.suid = pIter->bData.suid; - pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; - pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); - } else { - pMerger->pNode = NULL; - } - } - - SRBTreeNode *pMinNode = tRBTreeMin(&pMerger->rbt); - if (pMerger->pNode && pMinNode) { - int32_t c = tRowInfoCmprFn(pMerger->pNode->payload, pMinNode->payload); - if (c > 0) { - pMerger->pNode = tRBTreePut(&pMerger->rbt, pMerger->pNode); - ASSERT(pMerger->pNode); - pMerger->pNode = NULL; - } else { - ASSERT(c); - } - } - } - - if (pMerger->pNode == NULL) { - pMerger->pNode = tRBTreeMin(&pMerger->rbt); - if (pMerger->pNode) { - tRBTreeDrop(&pMerger->rbt, pMerger->pNode); - } - } - - if (pMerger->pNode) { - *ppInfo = &((SLDataIter *)pMerger->pNode->payload)[0].rowInfo; - } else { - *ppInfo = NULL; - } - -_exit: - return code; -} - -typedef struct { - STsdb *pTsdb; - int8_t maxLast; - int32_t minRow; - int32_t maxRow; - int8_t cmprAlg; - int64_t commitID; - STsdbFS fs; - struct { - SDataFReader *pReader; - SArray *aBlockIdx; - SLDataIter *aLDataiter[TSDB_MAX_LAST_FILE]; - SDataMerger merger; - } dReader; - struct { - SDataFWriter *pWriter; - SArray *aBlockIdx; - SArray *aBlockL; - SBlockData bData; - SBlockData bDatal; - } dWriter; -} STsdbMerger; - -static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { - int32_t code = 0; - STsdb *pTsdb = pMerger->pTsdb; - - // reader - code = tsdbDataFReaderOpen(&pMerger->dReader.pReader, pTsdb, pSet); - if (code) goto _err; - - code = tsdbReadBlockIdx(pMerger->dReader.pReader, pMerger->dReader.aBlockIdx); - if (code) goto _err; - - pMerger->dReader.merger.pNode = NULL; - tRBTreeCreate(&pMerger->dReader.merger.rbt, tRowInfoCmprFn); - for (int8_t iLast = 0; iLast < pSet->nLastF; iLast++) { - SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryCalloc(1, sizeof(*pNode) + sizeof(SLDataIter)); - if (pNode == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - SLDataIter *pIter = (SLDataIter *)pNode->payload; - pIter->pReader = pMerger->dReader.pReader; - pIter->iLast = iLast; - - pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pIter->aBlockL == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - code = tBlockDataCreate(&pIter->bData); - if (code) goto _err; - - code = tsdbReadBlockL(pMerger->dReader.pReader, iLast, pIter->aBlockL); - if (code) goto _err; - - if (taosArrayGetSize(pIter->aBlockL) == 0) continue; - pIter->iBlockL = 0; - - SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, 0); - code = tsdbReadLastBlockEx(pMerger->dReader.pReader, iLast, pBlockL, &pIter->bData); - if (code) goto _err; - - pIter->iRow = 0; - pIter->rowInfo.suid = pIter->bData.suid; - pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; - pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); - - pNode = tRBTreePut(&pMerger->dReader.merger.rbt, pNode); - ASSERT(pNode); - - pMerger->dReader.aLDataiter[iLast] = pIter; - } - - // writer - SHeadFile fHead = {.commitID = pMerger->commitID}; - SDataFile fData = *pSet->pDataF; - SSmaFile fSma = *pSet->pSmaF; - SLastFile fLast = {.commitID = pMerger->commitID}; - SDFileSet wSet = {.diskId = pSet->diskId, - .fid = pSet->fid, - .nLastF = 1, - .pHeadF = &fHead, - .pDataF = &fData, - .pSmaF = &fSma, - .aLastF[0] = &fLast}; - code = tsdbDataFWriterOpen(&pMerger->dWriter.pWriter, pTsdb, &wSet); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d tsdb merge file data start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbMergeFileDataEnd(STsdbMerger *pMerger) { - int32_t code = 0; - STsdb *pTsdb = pMerger->pTsdb; - - // write aBlockIdx - code = tsdbWriteBlockIdx(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockIdx); - if (code) goto _err; - - // write aBlockL - code = tsdbWriteBlockL(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockL); - if (code) goto _err; - - // update file header - code = tsdbUpdateDFileSetHeader(pMerger->dWriter.pWriter); - if (code) goto _err; - - // upsert SDFileSet - code = tsdbFSUpsertFSet(&pMerger->fs, &pMerger->dWriter.pWriter->wSet); - if (code) goto _err; - - // close and sync - code = tsdbDataFWriterClose(&pMerger->dWriter.pWriter, 1); - if (code) goto _err; - - if (pMerger->dReader.pReader) { - code = tsdbDataFReaderClose(&pMerger->dReader.pReader); - if (code) goto _err; - } - - return code; - -_err: - tsdbError("vgId:%d tsdb merge file data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { - int32_t code = 0; - STsdb *pTsdb = pMerger->pTsdb; - - // start - code = tsdbMergeFileDataStart(pMerger, pSet); - if (code) goto _err; - - // impl - SRowInfo *pInfo; - TABLEID id = {0}; - while (true) { - code = tDataMergeNext(&pMerger->dReader.merger, &pInfo); - if (code) goto _err; - - if (pInfo == NULL) { - if (pMerger->dWriter.bData.nRow > 0) { - // TODO - } - - if (pMerger->dWriter.bDatal.nRow > 0) { - // TODO - } - - break; - } - - if (id.suid != pInfo->suid || id.uid != pInfo->uid) { - while (true) { - // move commit the head data - } - - // prepare to commit next - } - - code = tBlockDataAppendRow(&pMerger->dWriter.bData, &pInfo->row, NULL, pInfo->uid); - if (code) goto _err; - - if (pMerger->dWriter.bData.nRow >= pMerger->maxRow * 4 / 5) { - // code = tsdbCommitDataBlock(); - if (code) goto _err; - } - } - - // end - code = tsdbMergeFileDataEnd(pMerger); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d tsdb merge file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbStartMerge(STsdbMerger *pMerger, STsdb *pTsdb) { - int32_t code = 0; - - pMerger->pTsdb = pTsdb; - pMerger->maxLast = TSDB_DEFAULT_LAST_FILE; - pMerger->commitID = ++pTsdb->pVnode->state.commitID; - code = tsdbFSCopy(pTsdb, &pMerger->fs); - if (code) goto _exit; - - // reader - pMerger->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pMerger->dReader.aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { - // pMerger->dReader.aBlockL[iLast] = taosArrayInit(0, sizeof(SBlockL)); - // if (pMerger->dReader.aBlockL[iLast] == NULL) { - // code = TSDB_CODE_OUT_OF_MEMORY; - // goto _exit; - // } - // } - - // writer - pMerger->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pMerger->dWriter.aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - pMerger->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pMerger->dWriter.aBlockL == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - -_exit: - return code; -} - -static int32_t tsdbEndMerge(STsdbMerger *pMerger) { - int32_t code = 0; - STsdb *pTsdb = pMerger->pTsdb; - - code = tsdbFSCommit1(pTsdb, &pMerger->fs); - if (code) goto _err; - - taosThreadRwlockWrlock(&pTsdb->rwLock); - code = tsdbFSCommit2(pTsdb, &pMerger->fs); - if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; - } - taosThreadRwlockUnlock(&pTsdb->rwLock); - - // writer - taosArrayDestroy(pMerger->dWriter.aBlockL); - taosArrayDestroy(pMerger->dWriter.aBlockIdx); - - // reader - // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { - // taosArrayDestroy(pMerger->dReader.aBlockL[iLast]); - // } - taosArrayDestroy(pMerger->dReader.aBlockIdx); - tsdbFSDestroy(&pMerger->fs); - - return code; - -_err: - tsdbError("vgId:%d, tsdb end merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbMerge(STsdb *pTsdb) { - int32_t code = 0; - STsdbMerger merger = {0}; - - code = tsdbStartMerge(&merger, pTsdb); - if (code) goto _err; - - for (int32_t iSet = 0; iSet < taosArrayGetSize(merger.fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(merger.fs.aDFileSet, iSet); - if (pSet->nLastF < merger.maxLast) continue; - - code = tsdbMergeFileData(&merger, pSet); - if (code) goto _err; - } - - code = tsdbEndMerge(&merger); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d tsdb merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} \ No newline at end of file +// // ================================================================================ +// typedef struct { +// SRowInfo rowInfo; +// SDataFReader *pReader; +// int32_t iLast; +// SArray *aBlockL; // SArray +// int32_t iBlockL; +// SBlockData bData; +// int32_t iRow; +// } SLDataIter; + +// typedef struct { +// SRBTreeNode *pNode; +// SRBTree rbt; +// } SDataMerger; + +// static void tDataMergerInit(SDataMerger *pMerger, SArray *aNodeP) { +// pMerger->pNode = NULL; +// tRBTreeCreate(&pMerger->rbt, tRowInfoCmprFn); +// for (int32_t iNode = 0; iNode < taosArrayGetSize(aNodeP); iNode++) { +// SRBTreeNode *pNode = (SRBTreeNode *)taosArrayGetP(aNodeP, iNode); + +// pNode = tRBTreePut(&pMerger->rbt, pNode); +// ASSERT(pNode); +// } +// } + +// static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { +// int32_t code = 0; + +// if (pMerger->pNode) { +// // next current iter +// SLDataIter *pIter = (SLDataIter *)pMerger->pNode->payload; + +// pIter->iRow++; +// if (pIter->iRow < pIter->bData.nRow) { +// pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; +// pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); +// } else { +// pIter->iBlockL++; +// if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { +// SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); +// code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pBlockL, &pIter->bData); +// if (code) goto _exit; + +// pIter->iRow = 0; +// pIter->rowInfo.suid = pIter->bData.suid; +// pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; +// pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); +// } else { +// pMerger->pNode = NULL; +// } +// } + +// SRBTreeNode *pMinNode = tRBTreeMin(&pMerger->rbt); +// if (pMerger->pNode && pMinNode) { +// int32_t c = tRowInfoCmprFn(pMerger->pNode->payload, pMinNode->payload); +// if (c > 0) { +// pMerger->pNode = tRBTreePut(&pMerger->rbt, pMerger->pNode); +// ASSERT(pMerger->pNode); +// pMerger->pNode = NULL; +// } else { +// ASSERT(c); +// } +// } +// } + +// if (pMerger->pNode == NULL) { +// pMerger->pNode = tRBTreeMin(&pMerger->rbt); +// if (pMerger->pNode) { +// tRBTreeDrop(&pMerger->rbt, pMerger->pNode); +// } +// } + +// if (pMerger->pNode) { +// *ppInfo = &((SLDataIter *)pMerger->pNode->payload)[0].rowInfo; +// } else { +// *ppInfo = NULL; +// } + +// _exit: +// return code; +// } + +// typedef struct { +// STsdb *pTsdb; +// int8_t maxLast; +// int32_t minRow; +// int32_t maxRow; +// int8_t cmprAlg; +// int64_t commitID; +// STsdbFS fs; +// struct { +// SDataFReader *pReader; +// SArray *aBlockIdx; +// SLDataIter *aLDataiter[TSDB_MAX_LAST_FILE]; +// SDataMerger merger; +// } dReader; +// struct { +// SDataFWriter *pWriter; +// SArray *aBlockIdx; +// SArray *aBlockL; +// SBlockData bData; +// SBlockData bDatal; +// } dWriter; +// } STsdbMerger; + +// static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { +// int32_t code = 0; +// STsdb *pTsdb = pMerger->pTsdb; + +// // reader +// code = tsdbDataFReaderOpen(&pMerger->dReader.pReader, pTsdb, pSet); +// if (code) goto _err; + +// code = tsdbReadBlockIdx(pMerger->dReader.pReader, pMerger->dReader.aBlockIdx); +// if (code) goto _err; + +// pMerger->dReader.merger.pNode = NULL; +// tRBTreeCreate(&pMerger->dReader.merger.rbt, tRowInfoCmprFn); +// for (int8_t iLast = 0; iLast < pSet->nLastF; iLast++) { +// SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryCalloc(1, sizeof(*pNode) + sizeof(SLDataIter)); +// if (pNode == NULL) { +// code = TSDB_CODE_OUT_OF_MEMORY; +// goto _err; +// } + +// SLDataIter *pIter = (SLDataIter *)pNode->payload; +// pIter->pReader = pMerger->dReader.pReader; +// pIter->iLast = iLast; + +// pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); +// if (pIter->aBlockL == NULL) { +// code = TSDB_CODE_OUT_OF_MEMORY; +// goto _err; +// } +// code = tBlockDataCreate(&pIter->bData); +// if (code) goto _err; + +// code = tsdbReadBlockL(pMerger->dReader.pReader, iLast, pIter->aBlockL); +// if (code) goto _err; + +// if (taosArrayGetSize(pIter->aBlockL) == 0) continue; +// pIter->iBlockL = 0; + +// SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, 0); +// code = tsdbReadLastBlockEx(pMerger->dReader.pReader, iLast, pBlockL, &pIter->bData); +// if (code) goto _err; + +// pIter->iRow = 0; +// pIter->rowInfo.suid = pIter->bData.suid; +// pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; +// pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); + +// pNode = tRBTreePut(&pMerger->dReader.merger.rbt, pNode); +// ASSERT(pNode); + +// pMerger->dReader.aLDataiter[iLast] = pIter; +// } + +// // writer +// SHeadFile fHead = {.commitID = pMerger->commitID}; +// SDataFile fData = *pSet->pDataF; +// SSmaFile fSma = *pSet->pSmaF; +// SLastFile fLast = {.commitID = pMerger->commitID}; +// SDFileSet wSet = {.diskId = pSet->diskId, +// .fid = pSet->fid, +// .nLastF = 1, +// .pHeadF = &fHead, +// .pDataF = &fData, +// .pSmaF = &fSma, +// .aLastF[0] = &fLast}; +// code = tsdbDataFWriterOpen(&pMerger->dWriter.pWriter, pTsdb, &wSet); +// if (code) goto _err; + +// return code; + +// _err: +// tsdbError("vgId:%d tsdb merge file data start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +// return code; +// } + +// static int32_t tsdbMergeFileDataEnd(STsdbMerger *pMerger) { +// int32_t code = 0; +// STsdb *pTsdb = pMerger->pTsdb; + +// // write aBlockIdx +// code = tsdbWriteBlockIdx(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockIdx); +// if (code) goto _err; + +// // write aBlockL +// code = tsdbWriteBlockL(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockL); +// if (code) goto _err; + +// // update file header +// code = tsdbUpdateDFileSetHeader(pMerger->dWriter.pWriter); +// if (code) goto _err; + +// // upsert SDFileSet +// code = tsdbFSUpsertFSet(&pMerger->fs, &pMerger->dWriter.pWriter->wSet); +// if (code) goto _err; + +// // close and sync +// code = tsdbDataFWriterClose(&pMerger->dWriter.pWriter, 1); +// if (code) goto _err; + +// if (pMerger->dReader.pReader) { +// code = tsdbDataFReaderClose(&pMerger->dReader.pReader); +// if (code) goto _err; +// } + +// return code; + +// _err: +// tsdbError("vgId:%d tsdb merge file data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +// return code; +// } + +// static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { +// int32_t code = 0; +// STsdb *pTsdb = pMerger->pTsdb; + +// // start +// code = tsdbMergeFileDataStart(pMerger, pSet); +// if (code) goto _err; + +// // impl +// SRowInfo *pInfo; +// TABLEID id = {0}; +// while (true) { +// code = tDataMergeNext(&pMerger->dReader.merger, &pInfo); +// if (code) goto _err; + +// if (pInfo == NULL) { +// if (pMerger->dWriter.bData.nRow > 0) { +// // TODO +// } + +// if (pMerger->dWriter.bDatal.nRow > 0) { +// // TODO +// } + +// break; +// } + +// if (id.suid != pInfo->suid || id.uid != pInfo->uid) { +// while (true) { +// // move commit the head data +// } + +// // prepare to commit next +// } + +// code = tBlockDataAppendRow(&pMerger->dWriter.bData, &pInfo->row, NULL, pInfo->uid); +// if (code) goto _err; + +// if (pMerger->dWriter.bData.nRow >= pMerger->maxRow * 4 / 5) { +// // code = tsdbCommitDataBlock(); +// if (code) goto _err; +// } +// } + +// // end +// code = tsdbMergeFileDataEnd(pMerger); +// if (code) goto _err; + +// return code; + +// _err: +// tsdbError("vgId:%d tsdb merge file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +// return code; +// } + +// static int32_t tsdbStartMerge(STsdbMerger *pMerger, STsdb *pTsdb) { +// int32_t code = 0; + +// pMerger->pTsdb = pTsdb; +// pMerger->maxLast = TSDB_DEFAULT_LAST_FILE; +// pMerger->commitID = ++pTsdb->pVnode->state.commitID; +// code = tsdbFSCopy(pTsdb, &pMerger->fs); +// if (code) goto _exit; + +// // reader +// pMerger->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); +// if (pMerger->dReader.aBlockIdx == NULL) { +// code = TSDB_CODE_OUT_OF_MEMORY; +// goto _exit; +// } +// // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { +// // pMerger->dReader.aBlockL[iLast] = taosArrayInit(0, sizeof(SBlockL)); +// // if (pMerger->dReader.aBlockL[iLast] == NULL) { +// // code = TSDB_CODE_OUT_OF_MEMORY; +// // goto _exit; +// // } +// // } + +// // writer +// pMerger->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); +// if (pMerger->dWriter.aBlockIdx == NULL) { +// code = TSDB_CODE_OUT_OF_MEMORY; +// goto _exit; +// } +// pMerger->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL)); +// if (pMerger->dWriter.aBlockL == NULL) { +// code = TSDB_CODE_OUT_OF_MEMORY; +// goto _exit; +// } + +// _exit: +// return code; +// } + +// static int32_t tsdbEndMerge(STsdbMerger *pMerger) { +// int32_t code = 0; +// STsdb *pTsdb = pMerger->pTsdb; + +// code = tsdbFSCommit1(pTsdb, &pMerger->fs); +// if (code) goto _err; + +// taosThreadRwlockWrlock(&pTsdb->rwLock); +// code = tsdbFSCommit2(pTsdb, &pMerger->fs); +// if (code) { +// taosThreadRwlockUnlock(&pTsdb->rwLock); +// goto _err; +// } +// taosThreadRwlockUnlock(&pTsdb->rwLock); + +// // writer +// taosArrayDestroy(pMerger->dWriter.aBlockL); +// taosArrayDestroy(pMerger->dWriter.aBlockIdx); + +// // reader +// // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { +// // taosArrayDestroy(pMerger->dReader.aBlockL[iLast]); +// // } +// taosArrayDestroy(pMerger->dReader.aBlockIdx); +// tsdbFSDestroy(&pMerger->fs); + +// return code; + +// _err: +// tsdbError("vgId:%d, tsdb end merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +// return code; +// } + +// int32_t tsdbMerge(STsdb *pTsdb) { +// int32_t code = 0; +// STsdbMerger merger = {0}; + +// code = tsdbStartMerge(&merger, pTsdb); +// if (code) goto _err; + +// for (int32_t iSet = 0; iSet < taosArrayGetSize(merger.fs.aDFileSet); iSet++) { +// SDFileSet *pSet = (SDFileSet *)taosArrayGet(merger.fs.aDFileSet, iSet); +// if (pSet->nLastF < merger.maxLast) continue; + +// code = tsdbMergeFileData(&merger, pSet); +// if (code) goto _err; +// } + +// code = tsdbEndMerge(&merger); +// if (code) goto _err; + +// return code; + +// _err: +// tsdbError("vgId:%d tsdb merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +// return code; +// } \ No newline at end of file From 3b9e03ba175eb250c8acd40794033be4e7385f69 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 16:56:42 +0800 Subject: [PATCH 042/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 491 +++++------------------ 1 file changed, 93 insertions(+), 398 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index b440a61469..ce31acc331 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1000,10 +1000,7 @@ _err: static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { int32_t code = 0; - // .data - while (true) { - if (pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) >= 0) break; - + while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) { SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); if (code) goto _err; @@ -1349,6 +1346,11 @@ static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row); } +static SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) { + // TODO + return NULL; +} + static int32_t tsdbNextCommitRow(SCommitter *pCommitter, SRowInfo **ppInfo) { int32_t code = 0; @@ -1437,6 +1439,66 @@ _exit: return code; } +static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { + int32_t code = 0; + SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx; + + ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0); + if (pBlockIdx && pBlockIdx->suid == id.suid && pBlockIdx->uid == id.uid) { + int32_t iBlock = 0; + SBlock block; + SBlock *pBlock = █ + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); + + ASSERT(pRowInfo->suid == id.suid && pRowInfo->uid == id.uid); + + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + while (pBlock && pRowInfo) { + SBlock tBlock = {.minKey = TSDBROW_KEY(&pRowInfo->row), .maxKey = TSDBROW_KEY(&pRowInfo->row)}; + int32_t c = tBlockCmprFn(pBlock, &tBlock); + + if (c < 0) { + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); + if (code) goto _err; + + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + } else { + pBlock = NULL; + } + } else if (c > 0) { + } else { + } + } + + while (pBlock) { + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); + if (code) goto _err; + + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + } else { + pBlock = NULL; + } + } + } + +_exit: + return code; + +_err: + tsdbError("vgId:%d tsdb merge table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbCommitTableData2(SCommitter *pCommitter, TABLEID id) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { int32_t code = 0; @@ -1447,42 +1509,45 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { if (code) goto _err; if (pRowInfo == NULL) { - // end the commit (todo) + /* end current table data commit (todo) */ + + /* end remain table data commit*/ + code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}); + if (code) goto _err; + + if (pCommitter->dWriter.bDatal.nRow > 0) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } + break; } if (id.suid != pRowInfo->suid || id.uid != pRowInfo->uid) { - // table changed, end current table commit (todo) + /* end current table data commit (todo) */ - // prepare the new + /* start new table data commit */ id.suid = pRowInfo->suid; id.uid = pRowInfo->uid; - } - - SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx; - if (pBlockIdx && pBlockIdx->suid == id.suid && pBlockIdx->uid == id.uid) { - while (true) { - /* code */ - } - } - - if (pRowInfo->row.type == 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, pRowInfo->suid, pRowInfo->uid, TSDBROW_SVERSION(&pRowInfo->row)); + // reader + code = tsdbMoveCommitData(pCommitter, id); + if (code) goto _err; + // writer + tMapDataReset(&pCommitter->dWriter.mBlock); + // other + code = tsdbCommitterUpdateTableSchema(pCommitter, id.suid, id.uid); + if (code) goto _err; + code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmRow.pTSchema); if (code) goto _err; } - code = tBlockDataAppendRow(&pCommitter->dWriter.bData, &pRowInfo->row, pCommitter->skmRow.pTSchema, pRowInfo->uid); + /* merge with data in .data file */ + code = tsdbMergeTableData(pCommitter, id); if (code) goto _err; - if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow * 4 / 5) { - if (1 /*toLastOnly*/) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } else { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - } - } + /* handle remain table data */ + code = tsdbCommitTableData2(pCommitter, id); + if (code) goto _err; } return code; @@ -1491,373 +1556,3 @@ _err: tsdbError("vgId:%d tsdb commit file data impl failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } - -// // ================================================================================ -// typedef struct { -// SRowInfo rowInfo; -// SDataFReader *pReader; -// int32_t iLast; -// SArray *aBlockL; // SArray -// int32_t iBlockL; -// SBlockData bData; -// int32_t iRow; -// } SLDataIter; - -// typedef struct { -// SRBTreeNode *pNode; -// SRBTree rbt; -// } SDataMerger; - -// static void tDataMergerInit(SDataMerger *pMerger, SArray *aNodeP) { -// pMerger->pNode = NULL; -// tRBTreeCreate(&pMerger->rbt, tRowInfoCmprFn); -// for (int32_t iNode = 0; iNode < taosArrayGetSize(aNodeP); iNode++) { -// SRBTreeNode *pNode = (SRBTreeNode *)taosArrayGetP(aNodeP, iNode); - -// pNode = tRBTreePut(&pMerger->rbt, pNode); -// ASSERT(pNode); -// } -// } - -// static int32_t tDataMergeNext(SDataMerger *pMerger, SRowInfo **ppInfo) { -// int32_t code = 0; - -// if (pMerger->pNode) { -// // next current iter -// SLDataIter *pIter = (SLDataIter *)pMerger->pNode->payload; - -// pIter->iRow++; -// if (pIter->iRow < pIter->bData.nRow) { -// pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; -// pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); -// } else { -// pIter->iBlockL++; -// if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { -// SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); -// code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pBlockL, &pIter->bData); -// if (code) goto _exit; - -// pIter->iRow = 0; -// pIter->rowInfo.suid = pIter->bData.suid; -// pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; -// pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); -// } else { -// pMerger->pNode = NULL; -// } -// } - -// SRBTreeNode *pMinNode = tRBTreeMin(&pMerger->rbt); -// if (pMerger->pNode && pMinNode) { -// int32_t c = tRowInfoCmprFn(pMerger->pNode->payload, pMinNode->payload); -// if (c > 0) { -// pMerger->pNode = tRBTreePut(&pMerger->rbt, pMerger->pNode); -// ASSERT(pMerger->pNode); -// pMerger->pNode = NULL; -// } else { -// ASSERT(c); -// } -// } -// } - -// if (pMerger->pNode == NULL) { -// pMerger->pNode = tRBTreeMin(&pMerger->rbt); -// if (pMerger->pNode) { -// tRBTreeDrop(&pMerger->rbt, pMerger->pNode); -// } -// } - -// if (pMerger->pNode) { -// *ppInfo = &((SLDataIter *)pMerger->pNode->payload)[0].rowInfo; -// } else { -// *ppInfo = NULL; -// } - -// _exit: -// return code; -// } - -// typedef struct { -// STsdb *pTsdb; -// int8_t maxLast; -// int32_t minRow; -// int32_t maxRow; -// int8_t cmprAlg; -// int64_t commitID; -// STsdbFS fs; -// struct { -// SDataFReader *pReader; -// SArray *aBlockIdx; -// SLDataIter *aLDataiter[TSDB_MAX_LAST_FILE]; -// SDataMerger merger; -// } dReader; -// struct { -// SDataFWriter *pWriter; -// SArray *aBlockIdx; -// SArray *aBlockL; -// SBlockData bData; -// SBlockData bDatal; -// } dWriter; -// } STsdbMerger; - -// static int32_t tsdbMergeFileDataStart(STsdbMerger *pMerger, SDFileSet *pSet) { -// int32_t code = 0; -// STsdb *pTsdb = pMerger->pTsdb; - -// // reader -// code = tsdbDataFReaderOpen(&pMerger->dReader.pReader, pTsdb, pSet); -// if (code) goto _err; - -// code = tsdbReadBlockIdx(pMerger->dReader.pReader, pMerger->dReader.aBlockIdx); -// if (code) goto _err; - -// pMerger->dReader.merger.pNode = NULL; -// tRBTreeCreate(&pMerger->dReader.merger.rbt, tRowInfoCmprFn); -// for (int8_t iLast = 0; iLast < pSet->nLastF; iLast++) { -// SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryCalloc(1, sizeof(*pNode) + sizeof(SLDataIter)); -// if (pNode == NULL) { -// code = TSDB_CODE_OUT_OF_MEMORY; -// goto _err; -// } - -// SLDataIter *pIter = (SLDataIter *)pNode->payload; -// pIter->pReader = pMerger->dReader.pReader; -// pIter->iLast = iLast; - -// pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); -// if (pIter->aBlockL == NULL) { -// code = TSDB_CODE_OUT_OF_MEMORY; -// goto _err; -// } -// code = tBlockDataCreate(&pIter->bData); -// if (code) goto _err; - -// code = tsdbReadBlockL(pMerger->dReader.pReader, iLast, pIter->aBlockL); -// if (code) goto _err; - -// if (taosArrayGetSize(pIter->aBlockL) == 0) continue; -// pIter->iBlockL = 0; - -// SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, 0); -// code = tsdbReadLastBlockEx(pMerger->dReader.pReader, iLast, pBlockL, &pIter->bData); -// if (code) goto _err; - -// pIter->iRow = 0; -// pIter->rowInfo.suid = pIter->bData.suid; -// pIter->rowInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; -// pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->bData, 0); - -// pNode = tRBTreePut(&pMerger->dReader.merger.rbt, pNode); -// ASSERT(pNode); - -// pMerger->dReader.aLDataiter[iLast] = pIter; -// } - -// // writer -// SHeadFile fHead = {.commitID = pMerger->commitID}; -// SDataFile fData = *pSet->pDataF; -// SSmaFile fSma = *pSet->pSmaF; -// SLastFile fLast = {.commitID = pMerger->commitID}; -// SDFileSet wSet = {.diskId = pSet->diskId, -// .fid = pSet->fid, -// .nLastF = 1, -// .pHeadF = &fHead, -// .pDataF = &fData, -// .pSmaF = &fSma, -// .aLastF[0] = &fLast}; -// code = tsdbDataFWriterOpen(&pMerger->dWriter.pWriter, pTsdb, &wSet); -// if (code) goto _err; - -// return code; - -// _err: -// tsdbError("vgId:%d tsdb merge file data start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); -// return code; -// } - -// static int32_t tsdbMergeFileDataEnd(STsdbMerger *pMerger) { -// int32_t code = 0; -// STsdb *pTsdb = pMerger->pTsdb; - -// // write aBlockIdx -// code = tsdbWriteBlockIdx(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockIdx); -// if (code) goto _err; - -// // write aBlockL -// code = tsdbWriteBlockL(pMerger->dWriter.pWriter, pMerger->dWriter.aBlockL); -// if (code) goto _err; - -// // update file header -// code = tsdbUpdateDFileSetHeader(pMerger->dWriter.pWriter); -// if (code) goto _err; - -// // upsert SDFileSet -// code = tsdbFSUpsertFSet(&pMerger->fs, &pMerger->dWriter.pWriter->wSet); -// if (code) goto _err; - -// // close and sync -// code = tsdbDataFWriterClose(&pMerger->dWriter.pWriter, 1); -// if (code) goto _err; - -// if (pMerger->dReader.pReader) { -// code = tsdbDataFReaderClose(&pMerger->dReader.pReader); -// if (code) goto _err; -// } - -// return code; - -// _err: -// tsdbError("vgId:%d tsdb merge file data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); -// return code; -// } - -// static int32_t tsdbMergeFileData(STsdbMerger *pMerger, SDFileSet *pSet) { -// int32_t code = 0; -// STsdb *pTsdb = pMerger->pTsdb; - -// // start -// code = tsdbMergeFileDataStart(pMerger, pSet); -// if (code) goto _err; - -// // impl -// SRowInfo *pInfo; -// TABLEID id = {0}; -// while (true) { -// code = tDataMergeNext(&pMerger->dReader.merger, &pInfo); -// if (code) goto _err; - -// if (pInfo == NULL) { -// if (pMerger->dWriter.bData.nRow > 0) { -// // TODO -// } - -// if (pMerger->dWriter.bDatal.nRow > 0) { -// // TODO -// } - -// break; -// } - -// if (id.suid != pInfo->suid || id.uid != pInfo->uid) { -// while (true) { -// // move commit the head data -// } - -// // prepare to commit next -// } - -// code = tBlockDataAppendRow(&pMerger->dWriter.bData, &pInfo->row, NULL, pInfo->uid); -// if (code) goto _err; - -// if (pMerger->dWriter.bData.nRow >= pMerger->maxRow * 4 / 5) { -// // code = tsdbCommitDataBlock(); -// if (code) goto _err; -// } -// } - -// // end -// code = tsdbMergeFileDataEnd(pMerger); -// if (code) goto _err; - -// return code; - -// _err: -// tsdbError("vgId:%d tsdb merge file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); -// return code; -// } - -// static int32_t tsdbStartMerge(STsdbMerger *pMerger, STsdb *pTsdb) { -// int32_t code = 0; - -// pMerger->pTsdb = pTsdb; -// pMerger->maxLast = TSDB_DEFAULT_LAST_FILE; -// pMerger->commitID = ++pTsdb->pVnode->state.commitID; -// code = tsdbFSCopy(pTsdb, &pMerger->fs); -// if (code) goto _exit; - -// // reader -// pMerger->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); -// if (pMerger->dReader.aBlockIdx == NULL) { -// code = TSDB_CODE_OUT_OF_MEMORY; -// goto _exit; -// } -// // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { -// // pMerger->dReader.aBlockL[iLast] = taosArrayInit(0, sizeof(SBlockL)); -// // if (pMerger->dReader.aBlockL[iLast] == NULL) { -// // code = TSDB_CODE_OUT_OF_MEMORY; -// // goto _exit; -// // } -// // } - -// // writer -// pMerger->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); -// if (pMerger->dWriter.aBlockIdx == NULL) { -// code = TSDB_CODE_OUT_OF_MEMORY; -// goto _exit; -// } -// pMerger->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL)); -// if (pMerger->dWriter.aBlockL == NULL) { -// code = TSDB_CODE_OUT_OF_MEMORY; -// goto _exit; -// } - -// _exit: -// return code; -// } - -// static int32_t tsdbEndMerge(STsdbMerger *pMerger) { -// int32_t code = 0; -// STsdb *pTsdb = pMerger->pTsdb; - -// code = tsdbFSCommit1(pTsdb, &pMerger->fs); -// if (code) goto _err; - -// taosThreadRwlockWrlock(&pTsdb->rwLock); -// code = tsdbFSCommit2(pTsdb, &pMerger->fs); -// if (code) { -// taosThreadRwlockUnlock(&pTsdb->rwLock); -// goto _err; -// } -// taosThreadRwlockUnlock(&pTsdb->rwLock); - -// // writer -// taosArrayDestroy(pMerger->dWriter.aBlockL); -// taosArrayDestroy(pMerger->dWriter.aBlockIdx); - -// // reader -// // for (int8_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { -// // taosArrayDestroy(pMerger->dReader.aBlockL[iLast]); -// // } -// taosArrayDestroy(pMerger->dReader.aBlockIdx); -// tsdbFSDestroy(&pMerger->fs); - -// return code; - -// _err: -// tsdbError("vgId:%d, tsdb end merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); -// return code; -// } - -// int32_t tsdbMerge(STsdb *pTsdb) { -// int32_t code = 0; -// STsdbMerger merger = {0}; - -// code = tsdbStartMerge(&merger, pTsdb); -// if (code) goto _err; - -// for (int32_t iSet = 0; iSet < taosArrayGetSize(merger.fs.aDFileSet); iSet++) { -// SDFileSet *pSet = (SDFileSet *)taosArrayGet(merger.fs.aDFileSet, iSet); -// if (pSet->nLastF < merger.maxLast) continue; - -// code = tsdbMergeFileData(&merger, pSet); -// if (code) goto _err; -// } - -// code = tsdbEndMerge(&merger); -// if (code) goto _err; - -// return code; - -// _err: -// tsdbError("vgId:%d tsdb merge failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); -// return code; -// } \ No newline at end of file From daf5d4239458d4c316b4af797a0713613d02a8d9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 17:05:17 +0800 Subject: [PATCH 043/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index ce31acc331..9af25338e5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1439,6 +1439,18 @@ _exit: return code; } +static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SBlock *pBlock) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SBlock *pBlock) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx; @@ -1468,7 +1480,23 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { pBlock = NULL; } } else if (c > 0) { + code = tsdbCommitAheadBlock(pCommitter, pBlock); + if (code) goto _err; + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; } else { + code = tsdbCommitMergeBlock(pCommitter, pBlock); + if (code) goto _err; + + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + } else { + pBlock = NULL; + } + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; } } From 712bc9a0ba0a6b42f9e3db8ad8c440bb111adf5f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 17:47:37 +0800 Subject: [PATCH 044/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 181 ++++++++++++++++++----- 1 file changed, 145 insertions(+), 36 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 9af25338e5..d0b1ca2bd9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -656,7 +656,8 @@ _err: } static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey) { - int32_t code = 0; + int32_t code = 0; +#if 0 STbData *pTbData = pIter->pTbData; SBlockData *pBlockData = &pCommitter->dWriter.bData; @@ -699,6 +700,7 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter _err: tsdbError("vgId:%d, tsdb commit table mem data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +#endif return code; } @@ -1346,12 +1348,11 @@ static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row); } -static SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) { - // TODO - return NULL; +static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) { + return (pCommitter->pIter) ? &pCommitter->pIter->r : NULL; } -static int32_t tsdbNextCommitRow(SCommitter *pCommitter, SRowInfo **ppInfo) { +static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { int32_t code = 0; if (pCommitter->pIter) { @@ -1429,25 +1430,139 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter, SRowInfo **ppInfo) { } } - if (pCommitter->pIter) { - *ppInfo = &pCommitter->pIter->r; - } else { - *ppInfo = NULL; - } - _exit: return code; } static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SBlock *pBlock) { - int32_t code = 0; - // TODO + int32_t code = 0; + SBlockData *pBlockData = &pCommitter->dWriter.bData; + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); + TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; + + tBlockDataClear(pBlockData); + while (pRowInfo) { + ASSERT(pRowInfo->row.type == 0); + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); + if (code) goto _err; + + code = tsdbNextCommitRow(pCommitter); + if (code) goto _err; + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo) { + if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) { + pRowInfo = NULL; + } else { + TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); + if (tsdbKeyCmprFn(&tKey, &pBlock->minKey) >= 0) pRowInfo = NULL; + } + } + + if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter, NULL); + if (code) goto _err; + } + } + + if (pBlockData->nRow) { + code = tsdbCommitDataBlock(pCommitter, NULL); + if (code) goto _err; + } + + return code; + +_err: + tsdbError("vgId:%d, tsdb commit ahead block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SBlock *pBlock) { - int32_t code = 0; - // TODO + int32_t code = 0; + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); + TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; + SBlockData *pBDataR = &pCommitter->dReader.bData; + SBlockData *pBDataW = &pCommitter->dWriter.bData; + + code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBDataR); + if (code) goto _err; + + tBlockDataClear(pBDataW); + int32_t iRow = 0; + TSDBROW row = tsdbRowFromBlockData(pBDataR, 0); + TSDBROW *pRow = &row; + + while (pRow && pRowInfo) { + int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); + if (c < 0) { + code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); + if (code) goto _err; + + iRow++; + if (iRow < pBDataR->nRow) { + row = tsdbRowFromBlockData(pBDataR, iRow); + } else { + pRow = NULL; + } + } else if (c > 0) { + ASSERT(pRowInfo->row.type == 0); + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); + if (code) goto _err; + + code = tsdbNextCommitRow(pCommitter); + if (code) goto _err; + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo) { + if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) { + pRowInfo = NULL; + } else { + TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); + if (tsdbKeyCmprFn(&tKey, &pBlock->maxKey) > 0) pRowInfo = NULL; + } + } + } else { + ASSERT(0); + } + + if (pBDataW->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter, NULL); + if (code) goto _err; + } + } + + while (pRow) { + code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); + if (code) goto _err; + + iRow++; + if (iRow < pBDataR->nRow) { + row = tsdbRowFromBlockData(pBDataR, iRow); + } else { + pRow = NULL; + } + + if (pBDataW->nRow >= pCommitter->maxRow * 4 / 5) { + code = tsdbCommitDataBlock(pCommitter, NULL); + if (code) goto _err; + } + } + + if (pBDataW->nRow) { + code = tsdbCommitDataBlock(pCommitter, NULL); + if (code) goto _err; + } + + return code; + +_err: + tsdbError("vgId:%d, tsdb commit merge block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } @@ -1533,12 +1648,8 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { SRowInfo *pRowInfo = NULL; TABLEID id = {0}; while (true) { - code = tsdbNextCommitRow(pCommitter, &pRowInfo); - if (code) goto _err; - + pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo == NULL) { - /* end current table data commit (todo) */ - /* end remain table data commit*/ code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}); if (code) goto _err; @@ -1551,23 +1662,21 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { break; } - if (id.suid != pRowInfo->suid || id.uid != pRowInfo->uid) { - /* end current table data commit (todo) */ + ASSERT(pRowInfo->suid != id.suid || pRowInfo->uid != id.uid); - /* start new table data commit */ - id.suid = pRowInfo->suid; - id.uid = pRowInfo->uid; - // reader - code = tsdbMoveCommitData(pCommitter, id); - if (code) goto _err; - // writer - tMapDataReset(&pCommitter->dWriter.mBlock); - // other - code = tsdbCommitterUpdateTableSchema(pCommitter, id.suid, id.uid); - if (code) goto _err; - code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmRow.pTSchema); - if (code) goto _err; - } + /* start new table data commit */ + id.suid = pRowInfo->suid; + id.uid = pRowInfo->uid; + // reader + code = tsdbMoveCommitData(pCommitter, id); + if (code) goto _err; + // writer + tMapDataReset(&pCommitter->dWriter.mBlock); + // other + code = tsdbCommitterUpdateTableSchema(pCommitter, id.suid, id.uid); + if (code) goto _err; + code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmRow.pTSchema); + if (code) goto _err; /* merge with data in .data file */ code = tsdbMergeTableData(pCommitter, id); From a224abef7e2458a33604cf7b449e1d9b6050bba5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 18:06:27 +0800 Subject: [PATCH 045/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 60 ++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d0b1ca2bd9..367fb4ac60 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1636,9 +1636,59 @@ _err: return code; } -static int32_t tsdbCommitTableData2(SCommitter *pCommitter, TABLEID id) { - int32_t code = 0; - // TODO +static int32_t tsdbCommitTableData2(SCommitter *pCommitter, TABLEID id, int8_t toLastOnly) { + int32_t code = 0; + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); + + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + + if (pRowInfo == NULL) goto _err; + +#if 0 + if (pBlockData->suid || pBlockData->uid) { + if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { + if (pBlockData->nRow > 0) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } + + tBlockDataReset(pBlockData); + } + } + + if (!pBlockData->suid && !pBlockData->uid) { + code = tBlockDataInit(pBlockData, pTbData->suid, 0, pCommitter->skmTable.pTSchema); + if (code) goto _err; + } +#endif + + SBlockData *pBlockData = NULL; // TODO + while (pRowInfo) { + STSchema *pTSchema = NULL; + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + if (code) goto _err; + + pTSchema = pCommitter->skmRow.pTSchema; + } + + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pTSchema, id.uid); + if (code) goto _err; + + code = tsdbNextCommitRow(pCommitter); + if (code) goto _err; + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + } + + return code; + +_err: return code; } @@ -1675,6 +1725,8 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { // other code = tsdbCommitterUpdateTableSchema(pCommitter, id.suid, id.uid); if (code) goto _err; + code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmRow.pTSchema); + if (code) goto _err; code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmRow.pTSchema); if (code) goto _err; @@ -1683,7 +1735,7 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { if (code) goto _err; /* handle remain table data */ - code = tsdbCommitTableData2(pCommitter, id); + code = tsdbCommitTableData2(pCommitter, id, 1); if (code) goto _err; } From efbbaa41200e7efb1684ba7ccffb9fc4f5fee14b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 18:25:02 +0800 Subject: [PATCH 046/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 367fb4ac60..b8af4d9d28 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -48,7 +48,6 @@ typedef struct { typedef struct { STsdb *pTsdb; - int8_t toMerge; /* commit data */ int64_t commitID; int32_t minutes; @@ -456,9 +455,6 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { wSet.nLastF = 1; wSet.aLastF[0] = &fLast; } - if (wSet.nLastF == pCommitter->maxLast) { - pCommitter->toMerge = 1; - } code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); if (code) goto _err; @@ -575,6 +571,7 @@ _err: return code; } +#if 0 static int32_t tsdbMergeCommitDataBlock(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { int32_t code = 0; STbData *pTbData = pIter->pTbData; @@ -657,7 +654,6 @@ _err: static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey) { int32_t code = 0; -#if 0 STbData *pTbData = pIter->pTbData; SBlockData *pBlockData = &pCommitter->dWriter.bData; @@ -700,7 +696,6 @@ static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter _err: tsdbError("vgId:%d, tsdb commit table mem data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); -#endif return code; } @@ -962,6 +957,7 @@ _err: tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } +#endif static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; @@ -1737,6 +1733,17 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { /* handle remain table data */ code = tsdbCommitTableData2(pCommitter, id, 1); if (code) goto _err; + + if (pCommitter->dWriter.mBlock.nItem > 0) { + SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid}; + code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); + if (code) goto _err; + + if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } } return code; From 7de0fc7dae79f18ca58037054f913d9f217ba3f8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 19:07:51 +0800 Subject: [PATCH 047/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 91 ++++++++++++++++++------ 1 file changed, 70 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index b8af4d9d28..d3b82db3e9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -32,7 +32,6 @@ typedef struct { int8_t type; union { struct { - SArray *aTbDataP; int32_t iTbDataP; STbDataIter iter; }; // memory data iter @@ -75,6 +74,7 @@ typedef struct { struct { SDataIter *pIter; SRBTree rbt; + SDataIter aDataIter[TSDB_MAX_LAST_FILE + 1]; }; struct { SDataFWriter *pWriter; @@ -102,6 +102,26 @@ static int32_t tsdbCommitData(SCommitter *pCommitter); static int32_t tsdbCommitDel(SCommitter *pCommitter); static int32_t tsdbCommitCache(SCommitter *pCommitter); static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno); +static int32_t tsdbNextCommitRow(SCommitter *pCommitter); + +static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { + SRowInfo *pInfo1 = (SRowInfo *)p1; + SRowInfo *pInfo2 = (SRowInfo *)p2; + + if (pInfo1->suid < pInfo2->suid) { + return -1; + } else if (pInfo1->suid > pInfo2->suid) { + return 1; + } + + if (pInfo1->uid < pInfo2->uid) { + return -1; + } else if (pInfo1->uid > pInfo2->uid) { + return 1; + } + + return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row); +} int32_t tsdbBegin(STsdb *pTsdb) { int32_t code = 0; @@ -376,6 +396,49 @@ _exit: return code; } +static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { + int32_t code = 0; + + tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); + pCommitter->pIter = NULL; + + int8_t iIter = 0; + // memory + SDataIter *pIter = &pCommitter->aDataIter[iIter]; + pIter->type = 0; + pIter->iTbDataP = 0; + for (; pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP); pIter->iTbDataP++) { + STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP); + TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN}; + tsdbTbDataIterOpen(pTbData, &tKey, 0, &pIter->iter); + TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter); + + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); + continue; + } + + pIter->r.suid = pTbData->suid; + pIter->r.uid = pTbData->uid; + pIter->r.row = *pRow; + break; + } + + tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); + + // disk + if (0) { + } + + code = tsdbNextCommitRow(pCommitter); + if (code) goto _err; + + return code; + +_err: + return code; +} + static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; @@ -464,6 +527,10 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { tBlockDataReset(&pCommitter->dWriter.bData); tBlockDataReset(&pCommitter->dWriter.bDatal); + // open iter + code = tsdbOpenCommitIter(pCommitter); + if (code) goto _err; + _exit: return code; @@ -1325,24 +1392,6 @@ _err: } // ================================================================================ -static int32_t tRowInfoCmprFn(const void *p1, const void *p2) { - SRowInfo *pInfo1 = (SRowInfo *)p1; - SRowInfo *pInfo2 = (SRowInfo *)p2; - - if (pInfo1->suid < pInfo2->suid) { - return -1; - } else if (pInfo1->suid > pInfo2->suid) { - return 1; - } - - if (pInfo1->uid < pInfo2->uid) { - return -1; - } else if (pInfo1->uid > pInfo2->uid) { - return 1; - } - - return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row); -} static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) { return (pCommitter->pIter) ? &pCommitter->pIter->r : NULL; @@ -1370,8 +1419,8 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { } pIter->iTbDataP++; - if (pIter->iTbDataP < taosArrayGetSize(pIter->aTbDataP)) { - STbData *pTbData = (STbData *)taosArrayGetP(pIter->aTbDataP, pIter->iTbDataP); + if (pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP)) { + STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP); TSDBKEY keyFrom = {.ts = pCommitter->minKey, .version = VERSION_MIN}; tsdbTbDataIterOpen(pTbData, &keyFrom, 0, &pIter->iter); pRow = tsdbTbDataIterGet(&pIter->iter); From 8ed246bdd32767dae49d94021a5b09b52c82d455 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 21:57:02 +0800 Subject: [PATCH 048/102] more code --- source/dnode/vnode/src/inc/tsdb.h | 11 ++++++ source/dnode/vnode/src/tsdb/tsdbCommit.c | 38 ++++++++++++++++--- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 11 ------ 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f5d4af9381..f3a2cd3079 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -609,6 +609,17 @@ struct STsdbReadSnap { STsdbFS fs; }; +struct SDataFReader { + STsdb *pTsdb; + SDFileSet *pSet; + TdFilePtr pHeadFD; + TdFilePtr pDataFD; + TdFilePtr pSmaFD; + TdFilePtr aLastFD[TSDB_MAX_LAST_FILE]; + + uint8_t *aBuf[3]; +}; + // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d3b82db3e9..2687911dcd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -26,10 +26,12 @@ typedef struct { TSDBROW row; } SRowInfo; +typedef enum { MEMORY_DATA_ITER = 0, LAST_DATA_ITER } EDataIterT; + typedef struct { SRBTreeNode n; SRowInfo r; - int8_t type; + EDataIterT type; union { struct { int32_t iTbDataP; @@ -74,7 +76,8 @@ typedef struct { struct { SDataIter *pIter; SRBTree rbt; - SDataIter aDataIter[TSDB_MAX_LAST_FILE + 1]; + SDataIter dataIter; + SDataIter aDataIter[TSDB_MAX_LAST_FILE]; }; struct { SDataFWriter *pWriter; @@ -402,10 +405,9 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); pCommitter->pIter = NULL; - int8_t iIter = 0; // memory - SDataIter *pIter = &pCommitter->aDataIter[iIter]; - pIter->type = 0; + SDataIter *pIter = &pCommitter->dataIter; + pIter->type = MEMORY_DATA_ITER; pIter->iTbDataP = 0; for (; pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP); pIter->iTbDataP++) { STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP); @@ -427,7 +429,31 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); // disk - if (0) { + SDataFReader *pReader = pCommitter->dReader.pReader; + if (pReader && pReader->pSet->nLastF >= pCommitter->maxLast) { + int8_t iIter = 0; + for (int32_t iLast = 0; iLast < pReader->pSet->nLastF; iLast++) { + pIter = &pCommitter->aDataIter[iIter]; + pIter->type = LAST_DATA_ITER; + pIter->iLast = iLast; + + code = tsdbReadBlockL(pCommitter->dReader.pReader, iLast, pIter->aBlockL); + if (code) goto _err; + + if (taosArrayGetSize(pIter->aBlockL) == 0) continue; + + pIter->iBlockL = 0; + SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, 0); + code = tsdbReadLastBlockEx(pCommitter->dReader.pReader, iLast, pBlockL, &pIter->bData); + if (code) goto _err; + + pIter->r.suid = pIter->bData.suid; + pIter->r.uid = pIter->bData.uid; + pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0); + + tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); + iIter++; + } } code = tsdbNextCommitRow(pCommitter); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 9c4a7aea13..664c7d1572 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -394,17 +394,6 @@ _err: } // SDataFReader ==================================================== -struct SDataFReader { - STsdb *pTsdb; - SDFileSet *pSet; - TdFilePtr pHeadFD; - TdFilePtr pDataFD; - TdFilePtr pSmaFD; - TdFilePtr aLastFD[TSDB_MAX_LAST_FILE]; - - uint8_t *aBuf[3]; -}; - int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; SDataFReader *pReader; From 8b58508e2e56ccf9572e53e2a59bfe30386592e0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 26 Aug 2022 22:23:03 +0800 Subject: [PATCH 049/102] more code --- include/util/trbtree.h | 1 + source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 33 +++++++++++++++++++++--- source/util/src/trbtree.c | 3 +++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 060d5a7918..50e2663648 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -56,6 +56,7 @@ struct SRBTreeNode { struct SRBTree { tRBTreeCmprFn cmprFn; + int64_t n; SRBTreeNode *root; SRBTreeNode *min; SRBTreeNode *max; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f3a2cd3079..9d4ff92b0b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -68,7 +68,7 @@ typedef struct SBlockCol SBlockCol; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_MAX_LAST_FILE 16 -#define TSDB_DEFAULT_LAST_FILE 8 +#define TSDB_DEFAULT_LAST_FILE 1 #define TSDB_FHDR_SIZE 512 #define HAS_NONE ((int8_t)0x1) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 2687911dcd..ce6cd9edc7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -78,6 +78,7 @@ typedef struct { SRBTree rbt; SDataIter dataIter; SDataIter aDataIter[TSDB_MAX_LAST_FILE]; + int8_t toLast; }; struct { SDataFWriter *pWriter; @@ -454,6 +455,10 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); iIter++; } + + pCommitter->toLast = 0; + } else { + pCommitter->toLast = 1; } code = tsdbNextCommitRow(pCommitter); @@ -1206,7 +1211,7 @@ _err: static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { int32_t code = 0; - // Reader + // reader pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCommitter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1216,7 +1221,20 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { code = tBlockDataCreate(&pCommitter->dReader.bData); if (code) goto _exit; - // Writer + // merger + for (int32_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { + SDataIter *pIter = &pCommitter->aDataIter[iLast]; + pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pIter->aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + code = tBlockDataCreate(&pIter->bData); + if (code) goto _exit; + } + + // writer pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCommitter->dWriter.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1240,12 +1258,19 @@ _exit: } static void tsdbCommitDataEnd(SCommitter *pCommitter) { - // Reader + // reader taosArrayDestroy(pCommitter->dReader.aBlockIdx); tMapDataClear(&pCommitter->dReader.mBlock); tBlockDataDestroy(&pCommitter->dReader.bData, 1); - // Writer + // merger + for (int32_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { + SDataIter *pIter = &pCommitter->aDataIter[iLast]; + taosArrayDestroy(pIter->aBlockL); + tBlockDataDestroy(&pIter->bData, 1); + } + + // writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); taosArrayDestroy(pCommitter->dWriter.aBlockL); tMapDataClear(&pCommitter->dWriter.mBlock); diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index ba011de0d7..2d81a6c5b4 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -201,6 +201,7 @@ static SRBTreeNode *tRBTreePredecessor(SRBTree *pTree, SRBTreeNode *pNode) { void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn) { pTree->cmprFn = cmprFn; + pTree->n = 0; pTree->NIL = &pTree->NILNODE; pTree->NIL->color = BLACK; pTree->NIL->parent = NULL; @@ -250,6 +251,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { if (pTree->max == pTree->NIL || pTree->cmprFn(pTree->max->payload, z->payload) < 0) { pTree->max = z; } + pTree->n++; return z; } @@ -294,6 +296,7 @@ void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z) { if (y_orignal_color == BLACK) { tRBTreeDropFix(pTree, x); } + pTree->n--; } SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { From f310c2cfcccc4b0221b8e0e96b695caf32d72da9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 27 Aug 2022 15:34:01 +0800 Subject: [PATCH 050/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index ce6cd9edc7..daee624f47 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1732,7 +1732,7 @@ _err: return code; } -static int32_t tsdbCommitTableData2(SCommitter *pCommitter, TABLEID id, int8_t toLastOnly) { +static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); @@ -1821,9 +1821,9 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { // other code = tsdbCommitterUpdateTableSchema(pCommitter, id.suid, id.uid); if (code) goto _err; - code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmRow.pTSchema); + code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); if (code) goto _err; - code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmRow.pTSchema); + code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); if (code) goto _err; /* merge with data in .data file */ @@ -1831,7 +1831,7 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { if (code) goto _err; /* handle remain table data */ - code = tsdbCommitTableData2(pCommitter, id, 1); + code = tsdbCommitTableData(pCommitter, id); if (code) goto _err; if (pCommitter->dWriter.mBlock.nItem > 0) { From d31459dc08a7bcdac8bc4fcb636567a0168edb1f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 27 Aug 2022 17:11:00 +0800 Subject: [PATCH 051/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 120 +++++++++++------------ 1 file changed, 55 insertions(+), 65 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index daee624f47..59e5d36064 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -78,7 +78,7 @@ typedef struct { SRBTree rbt; SDataIter dataIter; SDataIter aDataIter[TSDB_MAX_LAST_FILE]; - int8_t toLast; + int8_t toLastOnly; }; struct { SDataFWriter *pWriter; @@ -403,30 +403,31 @@ _exit: static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { int32_t code = 0; - tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); pCommitter->pIter = NULL; + tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); // memory + TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN}; SDataIter *pIter = &pCommitter->dataIter; pIter->type = MEMORY_DATA_ITER; pIter->iTbDataP = 0; for (; pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP); pIter->iTbDataP++) { STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP); - TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN}; tsdbTbDataIterOpen(pTbData, &tKey, 0, &pIter->iter); TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); - continue; + pRow = NULL; } + if (pRow == NULL) continue; + pIter->r.suid = pTbData->suid; pIter->r.uid = pTbData->uid; pIter->r.row = *pRow; break; } - + ASSERT(pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP)); tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); // disk @@ -448,17 +449,22 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { code = tsdbReadLastBlockEx(pCommitter->dReader.pReader, iLast, pBlockL, &pIter->bData); if (code) goto _err; + pIter->iRow = 0; pIter->r.suid = pIter->bData.suid; - pIter->r.uid = pIter->bData.uid; + pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0); tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); iIter++; } - pCommitter->toLast = 0; + if (iIter > 0) { + pCommitter->toLastOnly = 1; + } else { + pCommitter->toLastOnly = 0; + } } else { - pCommitter->toLast = 1; + pCommitter->toLastOnly = 0; } code = tsdbNextCommitRow(pCommitter); @@ -482,8 +488,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { pCommitter->nextKey = TSKEY_MAX; // Reader - pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &(SDFileSet){.fid = pCommitter->commitFid}, - tDFileSetCmprFn, TD_EQ); + SDFileSet tDFileSet = {.fid = pCommitter->commitFid}; + pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ); if (pRSet) { code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet); if (code) goto _err; @@ -493,10 +499,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { if (code) goto _err; pCommitter->dReader.iBlockIdx = 0; - if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { - pCommitter->dReader.pBlockIdx = - (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - + if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) { + pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0); code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); if (code) goto _err; } else { @@ -508,47 +512,32 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { } // Writer - SHeadFile fHead; - SDataFile fData; - SSmaFile fSma; - SLastFile fLast; - SDFileSet wSet = {0}; + SHeadFile fHead = {.commitID = pCommitter->commitID}; + SDataFile fData = {.commitID = pCommitter->commitID}; + SSmaFile fSma = {.commitID = pCommitter->commitID}; + SLastFile fLast = {.commitID = pCommitter->commitID}; + SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma}; if (pRSet) { - ASSERT(pCommitter->maxLast == 1 || pRSet->nLastF < pCommitter->maxLast); - - fHead = (SHeadFile){.commitID = pCommitter->commitID}; + ASSERT(pRSet->nLastF <= pCommitter->maxLast); fData = *pRSet->pDataF; fSma = *pRSet->pSmaF; - fLast = (SLastFile){.commitID = pCommitter->commitID}; - wSet.diskId = pRSet->diskId; - wSet.fid = pCommitter->commitFid; - wSet.pHeadF = &fHead; - wSet.pDataF = &fData; - wSet.pSmaF = &fSma; - for (int8_t iLast = 0; iLast < pRSet->nLastF; iLast++) { - wSet.aLastF[iLast] = pRSet->aLastF[iLast]; + if (pRSet->nLastF < pCommitter->maxLast) { + for (int32_t iLast = 0; iLast < pRSet->nLastF; iLast++) { + wSet.aLastF[iLast] = pRSet->aLastF[iLast]; + } + wSet.nLastF = pRSet->nLastF + 1; + } else { + wSet.nLastF = 1; } - wSet.nLastF = pRSet->nLastF + 1; - wSet.aLastF[wSet.nLastF - 1] = &fLast; // todo } else { - fHead = (SHeadFile){.commitID = pCommitter->commitID}; - fData = (SDataFile){.commitID = pCommitter->commitID}; - fSma = (SSmaFile){.commitID = pCommitter->commitID}; - fLast = (SLastFile){.commitID = pCommitter->commitID}; - SDiskID did = {0}; tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did); tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); - wSet.diskId = did; - wSet.fid = pCommitter->commitFid; - wSet.pHeadF = &fHead; - wSet.pDataF = &fData; - wSet.pSmaF = &fSma; wSet.nLastF = 1; - wSet.aLastF[0] = &fLast; } + wSet.aLastF[wSet.nLastF - 1] = &fLast; code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); if (code) goto _err; @@ -1722,6 +1711,9 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { pBlock = NULL; } } + + code = tsdbCommitterNextTableData(pCommitter); + if (code) goto _err; } _exit: @@ -1791,34 +1783,20 @@ _err: static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { int32_t code = 0; - SRowInfo *pRowInfo = NULL; + SRowInfo *pRowInfo; TABLEID id = {0}; - while (true) { - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo == NULL) { - /* end remain table data commit*/ - code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}); - if (code) goto _err; - - if (pCommitter->dWriter.bDatal.nRow > 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } - - break; - } - + while ((pRowInfo = tsdbGetCommitRow(pCommitter)) != NULL) { ASSERT(pRowInfo->suid != id.suid || pRowInfo->uid != id.uid); - - /* start new table data commit */ id.suid = pRowInfo->suid; id.uid = pRowInfo->uid; - // reader + code = tsdbMoveCommitData(pCommitter, id); if (code) goto _err; - // writer + + // start tMapDataReset(&pCommitter->dWriter.mBlock); - // other + + // impl code = tsdbCommitterUpdateTableSchema(pCommitter, id.suid, id.uid); if (code) goto _err; code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); @@ -1834,6 +1812,7 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { code = tsdbCommitTableData(pCommitter, id); if (code) goto _err; + // end if (pCommitter->dWriter.mBlock.nItem > 0) { SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid}; code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); @@ -1846,6 +1825,17 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { } } + id.suid = INT64_MAX; + id.uid = INT64_MAX; + code = tsdbMoveCommitData(pCommitter, id); + if (code) goto _err; + + // TODO: here may have problem + if (pCommitter->dWriter.bDatal.nRow > 0) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } + return code; _err: From 92ca817b9941949cd9c0c224a378fc33ec5132d8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 28 Aug 2022 00:00:48 +0800 Subject: [PATCH 052/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 182 ++++++++++++++++------- 1 file changed, 132 insertions(+), 50 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 59e5d36064..bb28c41e93 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -559,18 +559,14 @@ _err: return code; } -static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { +static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) { int32_t code = 0; SBlockData *pBlockData = &pCommitter->dWriter.bData; SBlock block; ASSERT(pBlockData->nRow > 0); - if (pBlock) { - block = *pBlock; // as a subblock - } else { - tBlockReset(&block); // as a new block - } + tBlockReset(&block); // info block.nRow += pBlockData->nRow; @@ -1547,14 +1543,14 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SBlock *pBlock) { } } - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); + if (pBlockData->nRow >= pCommitter->maxRow) { + code = tsdbCommitDataBlock(pCommitter); if (code) goto _err; } } if (pBlockData->nRow) { - code = tsdbCommitDataBlock(pCommitter, NULL); + code = tsdbCommitDataBlock(pCommitter); if (code) goto _err; } @@ -1616,8 +1612,8 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SBlock *pBlock) { ASSERT(0); } - if (pBDataW->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); + if (pBDataW->nRow >= pCommitter->maxRow) { + code = tsdbCommitDataBlock(pCommitter); if (code) goto _err; } } @@ -1633,14 +1629,14 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SBlock *pBlock) { pRow = NULL; } - if (pBDataW->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); + if (pBDataW->nRow >= pCommitter->maxRow) { + code = tsdbCommitDataBlock(pCommitter); if (code) goto _err; } } if (pBDataW->nRow) { - code = tsdbCommitDataBlock(pCommitter, NULL); + code = tsdbCommitDataBlock(pCommitter); if (code) goto _err; } @@ -1724,53 +1720,37 @@ _err: return code; } -static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { - int32_t code = 0; - SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); +static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { + int32_t code = 0; - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } + SBlockData *pBData = &pCommitter->dWriter.bData; + SBlockData *pBDatal = &pCommitter->dWriter.bDatal; - if (pRowInfo == NULL) goto _err; - -#if 0 - if (pBlockData->suid || pBlockData->uid) { - if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { - if (pBlockData->nRow > 0) { + if (pBDatal->suid || pBDatal->uid) { + if (pBDatal->suid != pBData->suid || pBDatal->suid == 0) { + if (pBDatal->nRow) { code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; } - - tBlockDataReset(pBlockData); + tBlockDataReset(pBDatal); } } - if (!pBlockData->suid && !pBlockData->uid) { - code = tBlockDataInit(pBlockData, pTbData->suid, 0, pCommitter->skmTable.pTSchema); + if (!pBDatal->suid && !pBDatal->uid) { + ASSERT(pCommitter->skmTable.suid == pBData->suid); + ASSERT(pCommitter->skmTable.uid == pBData->uid); + code = tBlockDataInit(pBDatal, pBData->suid, 0, pCommitter->skmTable.pTSchema); if (code) goto _err; } -#endif - SBlockData *pBlockData = NULL; // TODO - while (pRowInfo) { - STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { + TSDBROW row = tsdbRowFromBlockData(pBData, iRow); + code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid); + if (code) goto _err; + + if (pBDatal->nRow >= pCommitter->maxRow) { + code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; - - pTSchema = pCommitter->skmRow.pTSchema; - } - - code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pTSchema, id.uid); - if (code) goto _err; - - code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; } } @@ -1780,6 +1760,109 @@ _err: return code; } +static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { + int32_t code = 0; + + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + + if (pRowInfo == NULL) goto _exit; + + if (pCommitter->toLastOnly) { + SBlockData *pBDatal = &pCommitter->dWriter.bDatal; + + if (pBDatal->suid || pBDatal->uid) { + if (pBDatal->suid != id.suid || pBDatal->suid == 0) { + if (pBDatal->nRow) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } + tBlockDataReset(pBDatal); + } + } + + if (!pBDatal->suid && !pBDatal->uid) { + ASSERT(pCommitter->skmTable.suid == id.suid); + ASSERT(pCommitter->skmTable.uid == id.uid); + code = tBlockDataInit(pBDatal, id.suid, 0, pCommitter->skmTable.pTSchema); + if (code) goto _err; + } + + while (pRowInfo) { + STSchema *pTSchema = NULL; + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + if (code) goto _err; + pTSchema = pCommitter->skmRow.pTSchema; + } + + code = tBlockDataAppendRow(pBDatal, &pRowInfo->row, pTSchema, id.uid); + if (code) goto _err; + + code = tsdbNextCommitRow(pCommitter); + if (code) goto _err; + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + + if (pBDatal->nRow >= pCommitter->maxRow) { + code = tsdbCommitLastBlock(pCommitter); + if (code) goto _err; + } + } + } else { + SBlockData *pBData = &pCommitter->dWriter.bData; + + ASSERT(pBData->nRow == 0); + + while (pRowInfo) { + STSchema *pTSchema = NULL; + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + if (code) goto _err; + pTSchema = pCommitter->skmRow.pTSchema; + } + + code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); + if (code) goto _err; + + code = tsdbNextCommitRow(pCommitter); + if (code) goto _err; + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + + if (pBData->nRow >= pCommitter->maxRow) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; + } + } + + if (pBData->nRow) { + if (pBData->nRow > pCommitter->minRow) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; + } else { + code = tsdbAppendLastBlock(pCommitter); + if (code) goto _err; + } + } + } + +_exit: + return code; + +_err: + tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} + static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { int32_t code = 0; @@ -1830,7 +1913,6 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, id); if (code) goto _err; - // TODO: here may have problem if (pCommitter->dWriter.bDatal.nRow > 0) { code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; From de8944b62c1b2c00e53c1110a308c2c26db4b438 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 28 Aug 2022 00:19:24 +0800 Subject: [PATCH 053/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 48 ++++++++++++------------ 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index bb28c41e93..3e16e5de9b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1720,29 +1720,41 @@ _err: return code; } -static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { +static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; - SBlockData *pBData = &pCommitter->dWriter.bData; SBlockData *pBDatal = &pCommitter->dWriter.bDatal; - if (pBDatal->suid || pBDatal->uid) { - if (pBDatal->suid != pBData->suid || pBDatal->suid == 0) { + if (pBDatal->suid != id.suid || id.uid == 0) { if (pBDatal->nRow) { code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; + if (code) goto _exit; } tBlockDataReset(pBDatal); } } if (!pBDatal->suid && !pBDatal->uid) { - ASSERT(pCommitter->skmTable.suid == pBData->suid); - ASSERT(pCommitter->skmTable.uid == pBData->uid); - code = tBlockDataInit(pBDatal, pBData->suid, 0, pCommitter->skmTable.pTSchema); - if (code) goto _err; + ASSERT(pCommitter->skmTable.suid == id.suid); + ASSERT(pCommitter->skmTable.uid == id.uid); + code = tBlockDataInit(pBDatal, id.suid, 0, pCommitter->skmTable.pTSchema); + if (code) goto _exit; } +_exit: + return code; +} + +static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { + int32_t code = 0; + + SBlockData *pBData = &pCommitter->dWriter.bData; + SBlockData *pBDatal = &pCommitter->dWriter.bDatal; + + TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; + code = tsdbInitLastBlockIfNeed(pCommitter, id); + if (code) goto _err; + for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { TSDBROW row = tsdbRowFromBlockData(pBData, iRow); code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid); @@ -1773,22 +1785,8 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pCommitter->toLastOnly) { SBlockData *pBDatal = &pCommitter->dWriter.bDatal; - if (pBDatal->suid || pBDatal->uid) { - if (pBDatal->suid != id.suid || pBDatal->suid == 0) { - if (pBDatal->nRow) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } - tBlockDataReset(pBDatal); - } - } - - if (!pBDatal->suid && !pBDatal->uid) { - ASSERT(pCommitter->skmTable.suid == id.suid); - ASSERT(pCommitter->skmTable.uid == id.uid); - code = tBlockDataInit(pBDatal, id.suid, 0, pCommitter->skmTable.pTSchema); - if (code) goto _err; - } + code = tsdbInitLastBlockIfNeed(pCommitter, id); + if (code) goto _err; while (pRowInfo) { STSchema *pTSchema = NULL; From 0e50a9666856e8b6c28abf25eee48aab8468e2a3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 28 Aug 2022 00:24:11 +0800 Subject: [PATCH 054/102] refact code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 89 +++++++++--------------- 1 file changed, 34 insertions(+), 55 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3e16e5de9b..49847bb780 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1782,77 +1782,56 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pRowInfo == NULL) goto _exit; + SBlockData *pBData; if (pCommitter->toLastOnly) { - SBlockData *pBDatal = &pCommitter->dWriter.bDatal; - + pBData = &pCommitter->dWriter.bDatal; code = tsdbInitLastBlockIfNeed(pCommitter, id); if (code) goto _err; + } else { + pBData = &pCommitter->dWriter.bData; + ASSERT(pBData->nRow == 0); + } - while (pRowInfo) { - STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - if (code) goto _err; - pTSchema = pCommitter->skmRow.pTSchema; - } - - code = tBlockDataAppendRow(pBDatal, &pRowInfo->row, pTSchema, id.uid); + while (pRowInfo) { + STSchema *pTSchema = NULL; + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); if (code) goto _err; + pTSchema = pCommitter->skmRow.pTSchema; + } - code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); + if (code) goto _err; - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } + code = tsdbNextCommitRow(pCommitter); + if (code) goto _err; - if (pBDatal->nRow >= pCommitter->maxRow) { + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + + if (pBData->nRow >= pCommitter->maxRow) { + if (pCommitter->toLastOnly) { code = tsdbCommitLastBlock(pCommitter); if (code) goto _err; - } - } - } else { - SBlockData *pBData = &pCommitter->dWriter.bData; - - ASSERT(pBData->nRow == 0); - - while (pRowInfo) { - STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - if (code) goto _err; - pTSchema = pCommitter->skmRow.pTSchema; - } - - code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); - if (code) goto _err; - - code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } - - if (pBData->nRow >= pCommitter->maxRow) { - code = tsdbCommitDataBlock(pCommitter); - if (code) goto _err; - } - } - - if (pBData->nRow) { - if (pBData->nRow > pCommitter->minRow) { - code = tsdbCommitDataBlock(pCommitter); - if (code) goto _err; } else { - code = tsdbAppendLastBlock(pCommitter); + code = tsdbCommitDataBlock(pCommitter); if (code) goto _err; } } } + if (!pCommitter->toLastOnly && pBData->nRow) { + if (pBData->nRow > pCommitter->minRow) { + code = tsdbCommitDataBlock(pCommitter); + if (code) goto _err; + } else { + code = tsdbAppendLastBlock(pCommitter); + if (code) goto _err; + } + } + _exit: return code; From 6a5423d08dee6a6a77947465167611405f9cab99 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 28 Aug 2022 00:25:44 +0800 Subject: [PATCH 055/102] more --- source/dnode/vnode/src/inc/tsdb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 9d4ff92b0b..f3a2cd3079 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -68,7 +68,7 @@ typedef struct SBlockCol SBlockCol; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_MAX_LAST_FILE 16 -#define TSDB_DEFAULT_LAST_FILE 1 +#define TSDB_DEFAULT_LAST_FILE 8 #define TSDB_FHDR_SIZE 512 #define HAS_NONE ((int8_t)0x1) From 2d3762a546b6c45e5c64ab6f011b3074a2e48f24 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 28 Aug 2022 01:00:23 +0800 Subject: [PATCH 056/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 49847bb780..fcb58e91b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -459,9 +459,9 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { } if (iIter > 0) { - pCommitter->toLastOnly = 1; - } else { pCommitter->toLastOnly = 0; + } else { + pCommitter->toLastOnly = 1; } } else { pCommitter->toLastOnly = 0; From 1852c21b405fdf316ab3b9cfd4de68bcf30d5ab2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 28 Aug 2022 16:30:37 +0800 Subject: [PATCH 057/102] refact code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 417 ----------------------- 1 file changed, 417 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index fcb58e91b3..0824faf390 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -654,394 +654,6 @@ _err: return code; } -#if 0 -static int32_t tsdbMergeCommitDataBlock(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { - int32_t code = 0; - STbData *pTbData = pIter->pTbData; - SBlockData *pBlockDataR = &pCommitter->dReader.bData; - SBlockData *pBlockDataW = &pCommitter->dWriter.bData; - - code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR); - if (code) goto _err; - - tBlockDataClear(pBlockDataW); - int32_t iRow = 0; - TSDBROW row; - TSDBROW *pRow1 = tsdbTbDataIterGet(pIter); - TSDBROW *pRow2 = &row; - *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); - while (pRow1 && pRow2) { - int32_t c = tsdbRowCmprFn(pRow1, pRow2); - - if (c < 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow1)); - if (code) goto _err; - - code = tBlockDataAppendRow(pBlockDataW, pRow1, pCommitter->skmRow.pTSchema, pTbData->uid); - if (code) goto _err; - - // next - tsdbTbDataIterNext(pIter); - pRow1 = tsdbTbDataIterGet(pIter); - } else if (c > 0) { - code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid); - if (code) goto _err; - - iRow++; - if (iRow < pBlockDataR->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); - } else { - pRow2 = NULL; - } - } else { - ASSERT(0); - } - - // check - if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - } - } - - while (pRow2) { - code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid); - if (code) goto _err; - - iRow++; - if (iRow < pBlockDataR->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); - } else { - pRow2 = NULL; - } - - // check - if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - } - } - - // check - if (pBlockDataW->nRow > 0) { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb merge commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey) { - int32_t code = 0; - STbData *pTbData = pIter->pTbData; - SBlockData *pBlockData = &pCommitter->dWriter.bData; - - tBlockDataClear(pBlockData); - TSDBROW *pRow = tsdbTbDataIterGet(pIter); - while (true) { - if (pRow == NULL) { - if (pBlockData->nRow > 0) { - goto _write_block; - } else { - break; - } - } - - // update schema - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; - - // append - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); - if (code) goto _err; - - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow) { - TSDBKEY rowKey = TSDBROW_KEY(pRow); - if (tsdbKeyCmprFn(&rowKey, &toKey) >= 0) { - pRow = NULL; - } - } - - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - _write_block: - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - } - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb commit table mem data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) { - int32_t nRow = 0; - - STbDataIter iter = *pIter; - while (true) { - TSDBROW *pRow = tsdbTbDataIterGet(&iter); - if (pRow == NULL) break; - - int32_t c = tsdbKeyCmprFn(&TSDBROW_KEY(pRow), &key); - if (c < 0) { - nRow++; - tsdbTbDataIterNext(&iter); - } else if (c > 0) { - break; - } else { - ASSERT(0); - } - } - - return nRow; -} - -static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { - int32_t code = 0; - STbData *pTbData = pIter->pTbData; - SBlockData *pBlockData = &pCommitter->dWriter.bData; - - tBlockDataClear(pBlockData); - TSDBROW *pRow = tsdbTbDataIterGet(pIter); - while (true) { - if (pRow == NULL) break; - - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; - - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); - if (code) goto _err; - - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow) { - TSDBKEY rowKey = TSDBROW_KEY(pRow); - if (tsdbKeyCmprFn(&rowKey, &pBlock->maxKey) > 0) { - pRow = NULL; - } - } - } - - ASSERT(pBlockData->nRow > 0 && pBlock->nRow + pBlockData->nRow <= pCommitter->maxRow); - - code = tsdbCommitDataBlock(pCommitter, pBlock); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d, tsdb merge as subblock failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbCommitLastFile(SCommitter *pCommitter, STbDataIter *pIter) { - int32_t code = 0; - STbData *pTbData = pIter->pTbData; - SBlockData *pBlockData = &pCommitter->dWriter.bDatal; - TSDBROW *pRow = tsdbTbDataIterGet(pIter); - - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - - if (pRow == NULL) goto _exit; - - if (pBlockData->suid || pBlockData->uid) { - if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { - if (pBlockData->nRow > 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } - - tBlockDataReset(pBlockData); - } - } - - if (!pBlockData->suid && !pBlockData->uid) { - code = tBlockDataInit(pBlockData, pTbData->suid, 0, pCommitter->skmTable.pTSchema); - if (code) goto _err; - } - - while (pRow) { - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; - - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); - if (code) goto _err; - - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - - if (pBlockData->nRow >= pCommitter->maxRow) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } - } - -_exit: - return code; - -_err: - tsdbError("vgId:%d tsdb merge commit last failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter) { - int32_t code = 0; - STbData *pTbData = pIter->pTbData; - int32_t iBlock = 0; - SBlock block; - SBlock *pBlock = █ - TSDBROW *pRow = tsdbTbDataIterGet(pIter); - - if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - - while (pBlock && pRow) { - SBlock tBlock = {.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)}; - int32_t c = tBlockCmprFn(pBlock, &tBlock); - - if (c < 0) { - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); - if (code) goto _err; - - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } else if (c > 0) { - code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey); - if (code) goto _err; - - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - } else { - int32_t nOvlp = tsdbGetNumOfRowsLessThan(pIter, pBlock->maxKey); - - ASSERT(nOvlp > 0); - - if (pBlock->nRow + nOvlp <= pCommitter->maxRow && pBlock->nSubBlock < TSDB_MAX_SUBBLOCKS) { - code = tsdbMergeAsSubBlock(pCommitter, pIter, pBlock); - if (code) goto _err; - } else { - code = tsdbMergeCommitDataBlock(pCommitter, pIter, pBlock); - if (code) goto _err; - } - - // next - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } - } - - while (pBlock) { - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); - if (code) goto _err; - - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } - -_exit: - return code; - -_err: - return code; -} - -static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { - int32_t code = 0; - - ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0); - - // merge commit table data - STbDataIter iter = {0}; - TSDBROW *pRow; - - tMapDataReset(&pCommitter->dWriter.mBlock); - - tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, &iter); - pRow = tsdbTbDataIterGet(&iter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - if (pRow == NULL) { - if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) == 0) { - code = tMapDataCopy(&pCommitter->dReader.mBlock, &pCommitter->dWriter.mBlock); - if (code) goto _err; - } - goto _exit; - } - - code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid); - if (code) goto _err; - code = tBlockDataInit(&pCommitter->dReader.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; - code = tBlockDataInit(&pCommitter->dWriter.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; - - // commit data - code = tsdbMergeCommitData(pCommitter, &iter); - if (code) goto _err; - - // commit last - code = tsdbCommitLastFile(pCommitter, &iter); - if (code) goto _err; - -_exit: - if (pCommitter->dWriter.mBlock.nItem > 0) { - SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid}; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); - if (code) goto _err; - - if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - pRow = tsdbTbDataIterGet(&iter); - if (pRow) { - pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); - } - - return code; - -_err: - tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} -#endif - static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; @@ -1112,38 +724,9 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { code = tsdbCommitFileDataStart(pCommitter); if (code) goto _err; -#if 1 // impl code = tsdbCommitFileDataImpl(pCommitter); if (code) goto _err; -#else - // commit file data impl - for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pCommitter->aTbDataP); iTbData++) { - STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); - - // move commit until current (suid, uid) - code = tsdbMoveCommitData(pCommitter, *(TABLEID *)pTbData); - if (code) goto _err; - - // commit current table data - code = tsdbCommitTableData(pCommitter, pTbData); - if (code) goto _err; - - // move next reader table data if need - if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { - code = tsdbCommitterNextTableData(pCommitter); - if (code) goto _err; - } - } - - code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}); - if (code) goto _err; - - if (pCommitter->dWriter.bDatal.nRow > 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } -#endif // commit file data end code = tsdbCommitFileDataEnd(pCommitter); From fdea9db9d34d2eeb1a925b971eed39dbc0e23b0f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 Aug 2022 09:58:30 +0800 Subject: [PATCH 058/102] fix(query): fix macro definition error. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index e221936647..5ff8fd9674 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2477,7 +2477,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // note: the lastblock may be null here initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == DEFAULT_ROW_INDEX_VAL || + if (pScanInfo->indexInBlockL == INITIAL_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); } From e2f99ba072faf40bba5d9a2f9adad1676aeccbaf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Aug 2022 11:19:25 +0800 Subject: [PATCH 059/102] more code --- source/dnode/vnode/CMakeLists.txt | 1 + source/dnode/vnode/src/inc/tsdb.h | 6 + source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 - source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 133 ++++++++++++++++++++ 4 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbMergeTree.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 3489863f37..eb1fbac2c9 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -52,6 +52,7 @@ target_sources( "src/tsdb/tsdbDiskData.c" "src/tsdb/tsdbCompress.c" "src/tsdb/tsdbCompact.c" + "src/tsdb/tsdbMergeTree.c" # tq "src/tq/tq.c" diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f3a2cd3079..968cc7f297 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -620,6 +620,12 @@ struct SDataFReader { uint8_t *aBuf[3]; }; +typedef struct { + int64_t suid; + int64_t uid; + TSDBROW row; +} SRowInfo; + // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 0824faf390..b227e97df6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -20,12 +20,6 @@ typedef struct { STSchema *pTSchema; } SSkmInfo; -typedef struct { - int64_t suid; - int64_t uid; - TSDBROW row; -} SRowInfo; - typedef enum { MEMORY_DATA_ITER = 0, LAST_DATA_ITER } EDataIterT; typedef struct { diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c new file mode 100644 index 0000000000..1d51025331 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +// SLDataIter ================================================= +typedef struct { + SRBTreeNode node; + SBlockL *pBlockL; + SRowInfo *pRowInfo; + + SDataFReader *pReader; + int32_t iLast; + int8_t backward; + SArray *aBlockL; + int32_t iBlockL; + SBlockData bData; + int32_t iRow; + SRowInfo rInfo; +} SLDataIter; + +int32_t tLDataIterOpen(SLDataIter *pIter, SDataFReader *pReader, int32_t iLast, int8_t backward) { + int32_t code = 0; + + pIter->pReader = pReader; + pIter->iLast = iLast; + pIter->backward = backward; + + pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if (pIter->aBlockL == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + code = tBlockDataCreate(&pIter->bData); + if (code) goto _exit; + + code = tsdbReadBlockL(pReader, iLast, pIter->aBlockL); + if (code) goto _exit; + + if (backward) { + pIter->iBlockL = taosArrayGetSize(pIter->aBlockL) - 1; + } else { + pIter->iBlockL = 0; + } + +_exit: + return code; +} + +void tLDataIterClose(SLDataIter *pIter) { + tBlockDataDestroy(&pIter->bData, 1); + taosArrayDestroy(pIter->aBlockL); +} + +extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData); + +void tLDataIterNextBlock(SLDataIter *pIter) { + if (pIter->backward) { + pIter->iBlockL--; + } else { + pIter->iBlockL++; + } + + if (pIter->iBlockL >= 0 && pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { + pIter->pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); + } else { + pIter->pBlockL = NULL; + } +} + +int32_t tLDataIterNextRow(SLDataIter *pIter) { + int32_t code = 0; + + if (pIter->backward) { + pIter->iRow--; + if (pIter->iRow < 0) { + pIter->iBlockL--; + + if (pIter->iBlockL >= 0) { + code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, taosArrayGet(pIter->aBlockL, pIter->iBlockL), + &pIter->bData); + if (code) goto _exit; + } else { + // TODO: no more data here + } + } + } else { + pIter->iRow++; + if (pIter->iRow >= pIter->bData.nRow) { + pIter->iBlockL++; + if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { + code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, taosArrayGet(pIter->aBlockL, pIter->iBlockL), + &pIter->bData); + if (code) goto _exit; + } else { + // TODO: not more data + goto _exit; + } + } + } + + pIter->rInfo.suid = pIter->bData.suid; + pIter->rInfo.uid = pIter->bData.uid; + pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + +_exit: + return code; +} + +SRowInfo *tLDataIterGet(SLDataIter *pIter) { + // TODO + return NULL; +} + +// SMergeTree ================================================= +typedef struct { + int8_t backward; + SRBTreeNode *pNode; + SRBTree rbt; +} SMergeTree; From f19b0f050f7981b14d990a9347c94fdaedf612cf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Aug 2022 11:21:09 +0800 Subject: [PATCH 060/102] more code --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 1d51025331..c6a6166b79 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -131,3 +131,9 @@ typedef struct { SRBTreeNode *pNode; SRBTree rbt; } SMergeTree; + +void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward) { + pMTree->backward = backward; + pMTree->pNode = NULL; + tRBTreeCreate(&pMTree->rbt, NULL); +} From 3111fad1f986b986b35feb6e71c6748774175a6a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Aug 2022 11:27:38 +0800 Subject: [PATCH 061/102] more code --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index c6a6166b79..12f017adfa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -132,8 +132,20 @@ typedef struct { SRBTree rbt; } SMergeTree; +static FORCE_INLINE tLDataIterCmprFn(const void *p1, const void *p2) { + SLDataIter *pIter1 = (SLDataIter *)p1; + SLDataIter *pIter2 = (SLDataIter *)p2; + + // TODO + ASSERT(0); + + return 0; +} + void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward) { pMTree->backward = backward; pMTree->pNode = NULL; - tRBTreeCreate(&pMTree->rbt, NULL); + tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); } + +void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } \ No newline at end of file From 43f03d9eef8a8decbe09126698b29c2d55edc5c3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Aug 2022 11:28:06 +0800 Subject: [PATCH 062/102] fix compile error --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 12f017adfa..b2e7d01917 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -132,7 +132,7 @@ typedef struct { SRBTree rbt; } SMergeTree; -static FORCE_INLINE tLDataIterCmprFn(const void *p1, const void *p2) { +static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { SLDataIter *pIter1 = (SLDataIter *)p1; SLDataIter *pIter2 = (SLDataIter *)p2; From e73b4a68bb890ce9936a5deca8bb3319ab3d7a1b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Aug 2022 11:31:35 +0800 Subject: [PATCH 063/102] refact some code --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 31 ++++++++++----------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index b2e7d01917..af92908e11 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -84,31 +84,28 @@ void tLDataIterNextBlock(SLDataIter *pIter) { int32_t tLDataIterNextRow(SLDataIter *pIter) { int32_t code = 0; + int32_t iBlockL = pIter->iBlockL; if (pIter->backward) { pIter->iRow--; if (pIter->iRow < 0) { - pIter->iBlockL--; - - if (pIter->iBlockL >= 0) { - code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, taosArrayGet(pIter->aBlockL, pIter->iBlockL), - &pIter->bData); - if (code) goto _exit; - } else { - // TODO: no more data here - } + tLDataIterNextBlock(pIter); } } else { pIter->iRow++; if (pIter->iRow >= pIter->bData.nRow) { pIter->iBlockL++; - if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { - code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, taosArrayGet(pIter->aBlockL, pIter->iBlockL), - &pIter->bData); - if (code) goto _exit; - } else { - // TODO: not more data - goto _exit; - } + tLDataIterNextBlock(pIter); + } + } + + if (iBlockL != pIter->iBlockL) { + if (pIter->pBlockL) { + code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pIter->pBlockL, &pIter->bData); + if (code) goto _exit; + pIter->iRow = 0; + } else { + // no more data + goto _exit; } } From 97699559b6d5b9a64155c883edc0de2db5e276fa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Aug 2022 17:55:53 +0800 Subject: [PATCH 064/102] fix a coredump --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index b227e97df6..179eff1b4f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1314,7 +1314,7 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { if (!pBDatal->suid && !pBDatal->uid) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); - code = tBlockDataInit(pBDatal, id.suid, 0, pCommitter->skmTable.pTSchema); + code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema); if (code) goto _exit; } From 03992f770bc93ef0e65b8e4a8be4437ab6825d0b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 31 Aug 2022 22:04:14 +0800 Subject: [PATCH 065/102] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 179eff1b4f..c6312326bb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1015,7 +1015,7 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { if (pCommitter->pIter) { SDataIter *pIter = pCommitter->pIter; - if (pCommitter->pIter->type == 0) { // memory + if (pCommitter->pIter->type == MEMORY_DATA_ITER) { // memory tsdbTbDataIterNext(&pIter->iter); TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter); while (true) { @@ -1043,7 +1043,7 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { break; } } - } else if (pCommitter->pIter->type == 1) { // last file + } else if (pCommitter->pIter->type == LAST_DATA_ITER) { // last file pIter->iRow++; if (pIter->iRow < pIter->bData.nRow) { pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; From 1cfd3e74f94a57838755e80c7f51d50ddf631274 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Sep 2022 11:56:56 +0800 Subject: [PATCH 066/102] enh(query): support merge multiple last files. --- source/dnode/vnode/src/inc/tsdb.h | 25 ++ source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 246 +++++++++++++++++--- source/dnode/vnode/src/tsdb/tsdbRead.c | 135 +++++------ 3 files changed, 296 insertions(+), 110 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 968cc7f297..bca180b64b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -64,6 +64,7 @@ typedef struct STsdbReadSnap STsdbReadSnap; typedef struct SBlockInfo SBlockInfo; typedef struct SSmaInfo SSmaInfo; typedef struct SBlockCol SBlockCol; +typedef struct SVersionRange SVersionRange; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 @@ -306,6 +307,12 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode); int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); +struct SLDataIter; +int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iLast, int8_t backward, uint64_t uid, + STimeWindow *pTimeWindow, SVersionRange *pRange); +void tLDataIterClose(struct SLDataIter *pIter); +bool tLDataIterNextRow(struct SLDataIter *pIter); + // structs ======================= struct STsdbFS { SDelFile *pDelFile; @@ -329,6 +336,11 @@ struct TSDBKEY { TSKEY ts; }; +struct SVersionRange { + uint64_t minVer; + uint64_t maxVer; +}; + typedef struct SMemSkipListNode SMemSkipListNode; struct SMemSkipListNode { int8_t level; @@ -626,6 +638,19 @@ typedef struct { TSDBROW row; } SRowInfo; +typedef struct SMergeTree { + int8_t backward; + SRBTreeNode *pNode; + SRBTree rbt; + struct SLDataIter *pIter; + SDataFReader* pLFileReader; +} SMergeTree; + +void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange); +void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter); +bool tMergeTreeNext(SMergeTree* pMTree); +TSDBROW tMergeTreeGetRow(SMergeTree* pMTree); + // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index af92908e11..bf0e41f800 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -15,12 +15,12 @@ #include "tsdb.h" -// SLDataIter ================================================= -typedef struct { - SRBTreeNode node; - SBlockL *pBlockL; - SRowInfo *pRowInfo; +#define INITIAL_IROW_INDEX (-1) +// SLDataIter ================================================= +typedef struct SLDataIter { + SRBTreeNode node; + SBlockL *pBlockL; SDataFReader *pReader; int32_t iLast; int8_t backward; @@ -29,31 +29,54 @@ typedef struct { SBlockData bData; int32_t iRow; SRowInfo rInfo; + uint64_t uid; + STimeWindow timeWindow; + SVersionRange verRange; } SLDataIter; -int32_t tLDataIterOpen(SLDataIter *pIter, SDataFReader *pReader, int32_t iLast, int8_t backward) { +int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iLast, int8_t backward, uint64_t uid, + STimeWindow *pTimeWindow, SVersionRange *pRange) { int32_t code = 0; + *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); - pIter->pReader = pReader; - pIter->iLast = iLast; - pIter->backward = backward; - - pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pIter->aBlockL == NULL) { + (*pIter)->uid = uid; + (*pIter)->timeWindow = *pTimeWindow; + (*pIter)->verRange = *pRange; + (*pIter)->pReader = pReader; + (*pIter)->iLast = iLast; + (*pIter)->backward = backward; + (*pIter)->aBlockL = taosArrayInit(0, sizeof(SBlockL)); + if ((*pIter)->aBlockL == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - code = tBlockDataCreate(&pIter->bData); - if (code) goto _exit; + code = tBlockDataCreate(&(*pIter)->bData); + if (code) { + goto _exit; + } - code = tsdbReadBlockL(pReader, iLast, pIter->aBlockL); - if (code) goto _exit; + code = tsdbReadBlockL(pReader, iLast, (*pIter)->aBlockL); + if (code) { + goto _exit; + } - if (backward) { - pIter->iBlockL = taosArrayGetSize(pIter->aBlockL) - 1; - } else { - pIter->iBlockL = 0; + size_t size = taosArrayGetSize((*pIter)->aBlockL); + + // find the start block + // todo handle the desc + int32_t index = -1; + for(int32_t i = 0; i < size; ++i) { + SBlockL *p = taosArrayGet((*pIter)->aBlockL, i); + if (p->minUid <= uid && p->maxUid >= uid) { + index = i; + break; + } + } + + (*pIter)->iBlockL = index; + if (index != -1) { + (*pIter)->pBlockL = taosArrayGet((*pIter)->aBlockL, (*pIter)->iBlockL); } _exit: @@ -74,15 +97,93 @@ void tLDataIterNextBlock(SLDataIter *pIter) { pIter->iBlockL++; } - if (pIter->iBlockL >= 0 && pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { - pIter->pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); - } else { + // todo handle desc order check. + int32_t index = -1; + size_t size = taosArrayGetSize(pIter->aBlockL); + for(int32_t i = pIter->iBlockL; i < size; ++i) { + SBlockL *p = taosArrayGet(pIter->aBlockL, i); + if (p->minUid <= pIter->uid && p->maxUid >= pIter->uid) { + index = i; + break; + } + + if (p->minUid > pIter->uid) { + break; + } + } + + if (index == -1) { pIter->pBlockL = NULL; + } else { + pIter->pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); } } -int32_t tLDataIterNextRow(SLDataIter *pIter) { +static void findNextValidRow(SLDataIter* pIter) { + int32_t step = pIter->backward? -1:1; + + bool hasVal = false; + int32_t i = pIter->iRow; + for (; i < pIter->bData.nRow && i >= 0; i += step) { + if (pIter->bData.aUid != NULL) { + if (!pIter->backward) { + if (pIter->bData.aUid[i] < pIter->uid) { + continue; + } else if (pIter->bData.aUid[i] > pIter->uid) { + break; + } + } else { + if (pIter->bData.aUid[i] > pIter->uid) { + continue; + } else if (pIter->bData.aUid[i] < pIter->uid) { + break; + } + } + } + + int64_t ts = pIter->bData.aTSKEY[i]; + if (ts < pIter->timeWindow.skey) { + continue; + } + + int64_t ver = pIter->bData.aVersion[i]; + if (ver < pIter->verRange.minVer) { + continue; + } + + // no data any more, todo opt handle desc case + if (ts > pIter->timeWindow.ekey) { + continue; + } + + // todo opt handle desc case + if (ver > pIter->verRange.maxVer) { + continue; + } + + // todo handle delete soon +#if 0 + TSDBKEY k = {.ts = ts, .version = ver}; + if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) { + continue; + } +#endif + + hasVal = true; + break; + } + + pIter->iRow = (hasVal)? i:-1; +} + +bool tLDataIterNextRow(SLDataIter *pIter) { int32_t code = 0; + int32_t step = pIter->backward? -1:1; + + // no qualified last file block in current file, no need to fetch row + if (pIter->pBlockL == NULL) { + return false; + } int32_t iBlockL = pIter->iBlockL; if (pIter->backward) { @@ -91,18 +192,38 @@ int32_t tLDataIterNextRow(SLDataIter *pIter) { tLDataIterNextBlock(pIter); } } else { - pIter->iRow++; - if (pIter->iRow >= pIter->bData.nRow) { - pIter->iBlockL++; + if (pIter->bData.nRow == 0 && pIter->pBlockL != NULL) { // current block not loaded yet + code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pIter->pBlockL, &pIter->bData); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + pIter->iRow = (pIter->backward)? pIter->bData.nRow:-1; + } + + pIter->iRow += step; + findNextValidRow(pIter); + + if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) { tLDataIterNextBlock(pIter); + if (pIter->pBlockL == NULL) { // no more data + goto _exit; + } } } if (iBlockL != pIter->iBlockL) { if (pIter->pBlockL) { code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pIter->pBlockL, &pIter->bData); - if (code) goto _exit; - pIter->iRow = 0; + if (code) { + goto _exit; + } + + pIter->iRow = pIter->backward? (pIter->bData.nRow-1):0; + findNextValidRow(pIter); + if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) { + // todo try next block + } } else { // no more data goto _exit; @@ -114,7 +235,11 @@ int32_t tLDataIterNextRow(SLDataIter *pIter) { pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); _exit: - return code; + if (code != TSDB_CODE_SUCCESS) { + return false; + } else { + return pIter->pBlockL != NULL; + } } SRowInfo *tLDataIterGet(SLDataIter *pIter) { @@ -123,12 +248,6 @@ SRowInfo *tLDataIterGet(SLDataIter *pIter) { } // SMergeTree ================================================= -typedef struct { - int8_t backward; - SRBTreeNode *pNode; - SRBTree rbt; -} SMergeTree; - static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { SLDataIter *pIter1 = (SLDataIter *)p1; SLDataIter *pIter2 = (SLDataIter *)p2; @@ -139,10 +258,61 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { return 0; } -void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward) { +void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange) { pMTree->backward = backward; pMTree->pNode = NULL; + pMTree->pIter = NULL; tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); + + struct SLDataIter* pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; + for(int32_t i = 0; i < pFReader->pSet->nLastF; ++i) { // open all last file + /*int32_t code = */tLDataIterOpen(&pIterList[i], pFReader, i, 0, uid, pTimeWindow, pVerRange); + bool hasVal = tLDataIterNextRow(pIterList[i]); + if (hasVal) { + tMergeTreeAddIter(pMTree, pIterList[i]); + } + } } -void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } \ No newline at end of file +void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } + +bool tMergeTreeNext(SMergeTree* pMTree) { + int32_t code = TSDB_CODE_SUCCESS; + if (pMTree->pIter) { + SLDataIter *pIter = pMTree->pIter; + + bool hasVal = tLDataIterNextRow(pIter); + if (!hasVal) { + pMTree->pIter = NULL; + } + + // compare with min in RB Tree + pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); + if (pMTree->pIter && pIter) { + int32_t c = pMTree->rbt.cmprFn(&pMTree->pIter->rInfo.row, &pIter->rInfo.row); + if (c > 0) { + tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); + pMTree->pIter = NULL; + } else { + ASSERT(c); + } + } + } + + if (pMTree->pIter == NULL) { + pMTree->pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); + if (pMTree->pIter) { + tRBTreeDrop(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); + } + } + + return pMTree->pIter != NULL; +} + +TSDBROW tMergeTreeGetRow(SMergeTree* pMTree) { + return pMTree->pIter->rInfo.row; +} + +void tMergeTreeClose(SMergeTree* pMTree) { + +} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5ff8fd9674..b8c664cddb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -83,11 +83,6 @@ typedef struct SBlockLoadSuppInfo { char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. } SBlockLoadSuppInfo; -typedef struct SVersionRange { - uint64_t minVer; - uint64_t maxVer; -} SVersionRange; - typedef struct SLastBlockReader { SArray* pBlockL; int32_t currentBlockIndex; @@ -96,7 +91,7 @@ typedef struct SLastBlockReader { SVersionRange verRange; int32_t order; uint64_t uid; - int16_t* rowIndex; // row index ptr, usually from the STableBlockScanInfo->indexInBlockL + SMergeTree mergeTree; } SLastBlockReader; typedef struct SFilesetIter { @@ -352,7 +347,6 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, pLReader->order = pReader->order; pLReader->window = pReader->window; pLReader->verRange = pReader->verRange; - pLReader->currentBlockIndex = -1; int32_t code = tBlockDataCreate(&pLReader->lastBlockData); if (code != TSDB_CODE_SUCCESS) { @@ -1346,7 +1340,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { tRowMerge(&merge, &fRow1); } else { @@ -1375,7 +1369,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { tRowMerge(&merge, &fRow1); } else { @@ -1407,14 +1401,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, bool mergeBlockData) { - SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); STSRow* pTSRow = NULL; SRowMerger merge = {0}; - TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); - + TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); @@ -1548,7 +1540,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { tRowMerge(&merge, &fRow1); } else { @@ -1598,7 +1590,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { tRowMerge(&merge, &fRow1); } else { @@ -1803,10 +1795,13 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, int16_t* startPos) { - pLastBlockReader->uid = uid; - pLastBlockReader->rowIndex = startPos; +static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, int16_t* startPos, SDataFReader* pFReader) { + // the last block reader has been initialized for this table. + if (pLastBlockReader->uid == uid) { + return true; + } + pLastBlockReader->uid = uid; if (*startPos == -1) { if (ASCENDING_TRAVERSE(pLastBlockReader->order)) { // do nothing @@ -1814,19 +1809,24 @@ static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid *startPos = pLastBlockReader->lastBlockData.nRow; } } + + /*int32_t code = */ tMergeTreeOpen(&pLastBlockReader->mergeTree, (pLastBlockReader->order == TSDB_ORDER_DESC), + pFReader, uid, &pLastBlockReader->window, &pLastBlockReader->verRange); + bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); + return hasVal; } static void setAllRowsChecked(SLastBlockReader* pLastBlockReader) { - *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; +// *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; } static bool nextRowInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { - bool asc = ASCENDING_TRAVERSE(pLastBlockReader->order); - int32_t step = (asc) ? 1 : -1; - if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { - return false; - } +// if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { +// return false; +// } + return tMergeTreeNext(&pLastBlockReader->mergeTree); +#if 0 *(pLastBlockReader->rowIndex) += step; SBlockData* pBlockData = &pLastBlockReader->lastBlockData; @@ -1879,20 +1879,17 @@ static bool nextRowInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc // set all data is consumed in last block setAllRowsChecked(pLastBlockReader); return false; +#endif } static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { - SBlockData* pBlockData = &pLastBlockReader->lastBlockData; - return pBlockData->aTSKEY[*pLastBlockReader->rowIndex]; + TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBKEY key = TSDBROW_KEY(&row); + return key.ts; } static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { - if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { - return false; - } - - ASSERT(pLastBlockReader->lastBlockData.nRow > 0); - return true; + return pLastBlockReader->mergeTree.pIter != NULL; } int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, @@ -1985,6 +1982,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } } + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); // no data in last block and block, no need to proceed. @@ -2248,6 +2246,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return TSDB_CODE_SUCCESS; } +#if 0 static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { SArray* pBlocks = pLastBlockReader->pBlockL; @@ -2308,6 +2307,7 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STable return TSDB_CODE_SUCCESS; } +#endif static int32_t uidComparFunc(const void* p1, const void* p2) { uint64_t pu1 = *(uint64_t*)p1; @@ -2401,26 +2401,14 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { while (1) { // load the last data block of current table STableBlockScanInfo* pScanInfo = pStatus->pTableIter; - code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + // code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); + // if (code != TSDB_CODE_SUCCESS) { + // return code; + // } - if (pLastBlockReader->currentBlockIndex != -1) { - initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - int32_t index = pScanInfo->indexInBlockL; - - if (index == INITIAL_ROW_INDEX_VAL || index == pLastBlockReader->lastBlockData.nRow) { - bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); - if (!hasData) { // current table does not have rows in last block, try next table - bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); - if (!hasNexTable) { - return TSDB_CODE_SUCCESS; - } - continue; - } - } - } else { // no data in last block, try next table + bool hasVal = + initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL, pReader->pFileReader); + if (!hasVal) { bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); if (!hasNexTable) { return TSDB_CODE_SUCCESS; @@ -2428,6 +2416,26 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { continue; } + // int32_t index = pScanInfo->indexInBlockL; + + // if (index == INITIAL_ROW_INDEX_VAL || index == pLastBlockReader->lastBlockData.nRow) { + // bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); + // if (!hasData) { // current table does not have rows in last block, try next table + // bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); + // if (!hasNexTable) { + // return TSDB_CODE_SUCCESS; + // } + // continue; + // } + // } + // } else { // no data in last block, try next table + // bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); + // if (!hasNexTable) { + // return TSDB_CODE_SUCCESS; + // } + // continue; + // } + code = doBuildDataBlock(pReader); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2446,7 +2454,6 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { } static int32_t doBuildDataBlock(STsdbReader* pReader) { - TSDBKEY key = {0}; int32_t code = TSDB_CODE_SUCCESS; SBlock* pBlock = NULL; @@ -2466,22 +2473,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { pBlock = getCurrentBlock(pBlockIter); } - { - key = getCurrentKeyInBuf(pScanInfo, pReader); - - // load the last data block of current table - code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // note: the lastblock may be null here - initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == INITIAL_ROW_INDEX_VAL || - pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { - bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); - } - } + TSDBKEY key = getCurrentKeyInBuf(pScanInfo, pReader); if (pBlockInfo == NULL) { // build data block from last data file ASSERT(pBlockIter->numOfBlocks == 0); @@ -2594,7 +2586,6 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl } SLastBlockReader* pLReader = pReader->status.fileIter.pLastBlockReader; - pLReader->currentBlockIndex = -1; // set the correct start position according to the query time window initBlockDumpInfo(pReader, pBlockIter); @@ -2660,8 +2651,8 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { bool hasNext = blockIteratorNext(&pReader->status.blockIter); if (hasNext) { // check for the next block in the block accessed order list initBlockDumpInfo(pReader, pBlockIter); - } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > - 0) { // data blocks in current file are exhausted, let's try the next file now + } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0) { + // data blocks in current file are exhausted, let's try the next file now tBlockDataReset(&pReader->status.fileBlockData); resetDataBlockIterator(pBlockIter, pReader->order); goto _begin; @@ -3024,7 +3015,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc while (nextRowInLastBlock(pLastBlockReader, pScanInfo)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { - TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tRowMerge(pMerger, &fRow1); } else { break; From 28e520656b315949f0da67c05a1965242da32a0d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 1 Sep 2022 14:24:47 +0800 Subject: [PATCH 067/102] fix: commit bug --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 61 +++++++++++++----------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index c6312326bb..1df0b5954a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -332,7 +332,10 @@ static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t su int32_t code = 0; if (suid) { - if (pCommitter->skmTable.suid == suid) goto _exit; + if (pCommitter->skmTable.suid == suid) { + pCommitter->skmTable.uid = uid; + goto _exit; + } } else { if (pCommitter->skmTable.uid == uid) goto _exit; } @@ -425,40 +428,44 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); // disk + pCommitter->toLastOnly = 0; SDataFReader *pReader = pCommitter->dReader.pReader; - if (pReader && pReader->pSet->nLastF >= pCommitter->maxLast) { - int8_t iIter = 0; - for (int32_t iLast = 0; iLast < pReader->pSet->nLastF; iLast++) { - pIter = &pCommitter->aDataIter[iIter]; - pIter->type = LAST_DATA_ITER; - pIter->iLast = iLast; + if (pReader) { + if (pReader->pSet->nLastF >= pCommitter->maxLast) { + int8_t iIter = 0; + for (int32_t iLast = 0; iLast < pReader->pSet->nLastF; iLast++) { + pIter = &pCommitter->aDataIter[iIter]; + pIter->type = LAST_DATA_ITER; + pIter->iLast = iLast; - code = tsdbReadBlockL(pCommitter->dReader.pReader, iLast, pIter->aBlockL); - if (code) goto _err; + code = tsdbReadBlockL(pCommitter->dReader.pReader, iLast, pIter->aBlockL); + if (code) goto _err; - if (taosArrayGetSize(pIter->aBlockL) == 0) continue; + if (taosArrayGetSize(pIter->aBlockL) == 0) continue; - pIter->iBlockL = 0; - SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, 0); - code = tsdbReadLastBlockEx(pCommitter->dReader.pReader, iLast, pBlockL, &pIter->bData); - if (code) goto _err; + pIter->iBlockL = 0; + SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, 0); + code = tsdbReadLastBlockEx(pCommitter->dReader.pReader, iLast, pBlockL, &pIter->bData); + if (code) goto _err; - pIter->iRow = 0; - pIter->r.suid = pIter->bData.suid; - pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; - pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0); + pIter->iRow = 0; + pIter->r.suid = pIter->bData.suid; + pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; + pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0); - tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); - iIter++; - } - - if (iIter > 0) { - pCommitter->toLastOnly = 0; + tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); + iIter++; + } } else { - pCommitter->toLastOnly = 1; + pCommitter->toLastOnly = 0; + for (int32_t iLast = 0; iLast < pReader->pSet->nLastF; iLast++) { + SLastFile *pLastFile = pReader->pSet->aLastF[iLast]; + if (pLastFile->size > pLastFile->offset) { + pCommitter->toLastOnly = 1; + break; + } + } } - } else { - pCommitter->toLastOnly = 0; } code = tsdbNextCommitRow(pCommitter); From 12cbd1c8bbc63e8525bf2ff1754e9a88f59f11f1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 1 Sep 2022 14:25:24 +0800 Subject: [PATCH 068/102] refact --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 1df0b5954a..3c497d143f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -457,7 +457,6 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { iIter++; } } else { - pCommitter->toLastOnly = 0; for (int32_t iLast = 0; iLast < pReader->pSet->nLastF; iLast++) { SLastFile *pLastFile = pReader->pSet->aLastF[iLast]; if (pLastFile->size > pLastFile->offset) { From e65927bc5b6efe51a2c9a6bcef15f3a582a88f9c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Sep 2022 15:11:40 +0800 Subject: [PATCH 069/102] fix(query): fix bug in merge multiple last files. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 92 +++++++++++---------- source/dnode/vnode/src/tsdb/tsdbRead.c | 7 -- 2 files changed, 50 insertions(+), 49 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index bf0e41f800..a32721b2b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -15,8 +15,6 @@ #include "tsdb.h" -#define INITIAL_IROW_INDEX (-1) - // SLDataIter ================================================= typedef struct SLDataIter { SRBTreeNode node; @@ -64,13 +62,22 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t size_t size = taosArrayGetSize((*pIter)->aBlockL); // find the start block - // todo handle the desc int32_t index = -1; - for(int32_t i = 0; i < size; ++i) { - SBlockL *p = taosArrayGet((*pIter)->aBlockL, i); - if (p->minUid <= uid && p->maxUid >= uid) { - index = i; - break; + if (!backward) { // asc + for (int32_t i = 0; i < size; ++i) { + SBlockL *p = taosArrayGet((*pIter)->aBlockL, i); + if (p->minUid <= uid && p->maxUid >= uid) { + index = i; + break; + } + } + } else { // desc + for (int32_t i = size - 1; i >= 0; --i) { + SBlockL *p = taosArrayGet((*pIter)->aBlockL, i); + if (p->minUid <= uid && p->maxUid >= uid) { + index = i; + break; + } } } @@ -91,16 +98,12 @@ void tLDataIterClose(SLDataIter *pIter) { extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData); void tLDataIterNextBlock(SLDataIter *pIter) { - if (pIter->backward) { - pIter->iBlockL--; - } else { - pIter->iBlockL++; - } + int32_t step = pIter->backward? -1:1; + pIter->iBlockL += step; - // todo handle desc order check. int32_t index = -1; size_t size = taosArrayGetSize(pIter->aBlockL); - for(int32_t i = pIter->iBlockL; i < size; ++i) { + for(int32_t i = pIter->iBlockL; i < size && i >= 0; i += step) { SBlockL *p = taosArrayGet(pIter->aBlockL, i); if (p->minUid <= pIter->uid && p->maxUid >= pIter->uid) { index = i; @@ -186,29 +189,22 @@ bool tLDataIterNextRow(SLDataIter *pIter) { } int32_t iBlockL = pIter->iBlockL; - if (pIter->backward) { - pIter->iRow--; - if (pIter->iRow < 0) { - tLDataIterNextBlock(pIter); + if (pIter->bData.nRow == 0 && pIter->pBlockL != NULL) { // current block not loaded yet + code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pIter->pBlockL, &pIter->bData); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; } - } else { - if (pIter->bData.nRow == 0 && pIter->pBlockL != NULL) { // current block not loaded yet - code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pIter->pBlockL, &pIter->bData); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } - pIter->iRow = (pIter->backward)? pIter->bData.nRow:-1; - } - - pIter->iRow += step; - findNextValidRow(pIter); + pIter->iRow = (pIter->backward)? pIter->bData.nRow:-1; + } - if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) { - tLDataIterNextBlock(pIter); - if (pIter->pBlockL == NULL) { // no more data - goto _exit; - } + pIter->iRow += step; + findNextValidRow(pIter); + + if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) { + tLDataIterNextBlock(pIter); + if (pIter->pBlockL == NULL) { // no more data + goto _exit; } } @@ -249,13 +245,25 @@ SRowInfo *tLDataIterGet(SLDataIter *pIter) { // SMergeTree ================================================= static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { - SLDataIter *pIter1 = (SLDataIter *)p1; - SLDataIter *pIter2 = (SLDataIter *)p2; + SLDataIter *pIter1 = (SLDataIter *)(p1 - sizeof(SRBTreeNode)); + SLDataIter *pIter2 = (SLDataIter *)(p2 - sizeof(SRBTreeNode)); - // TODO - ASSERT(0); + TSDBKEY key1 = TSDBROW_KEY(&pIter1->rInfo.row); + TSDBKEY key2 = TSDBROW_KEY(&pIter2->rInfo.row); - return 0; + if (key1.ts < key2.ts) { + return -1; + } else if (key1.ts > key2.ts) { + return 1; + } else { + if (key1.version < key2.version) { + return -1; + } else if (key1.version > key2.version) { + return 1; + } else { + return 0; + } + } } void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange) { @@ -266,7 +274,7 @@ void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, struct SLDataIter* pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; for(int32_t i = 0; i < pFReader->pSet->nLastF; ++i) { // open all last file - /*int32_t code = */tLDataIterOpen(&pIterList[i], pFReader, i, 0, uid, pTimeWindow, pVerRange); + /*int32_t code = */tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); bool hasVal = tLDataIterNextRow(pIterList[i]); if (hasVal) { tMergeTreeAddIter(pMTree, pIterList[i]); @@ -289,7 +297,7 @@ bool tMergeTreeNext(SMergeTree* pMTree) { // compare with min in RB Tree pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); if (pMTree->pIter && pIter) { - int32_t c = pMTree->rbt.cmprFn(&pMTree->pIter->rInfo.row, &pIter->rInfo.row); + int32_t c = pMTree->rbt.cmprFn(pMTree->pIter->node.payload, &pIter->node.payload); if (c > 0) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); pMTree->pIter = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b8c664cddb..ec0dd8cec4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1816,14 +1816,7 @@ static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid return hasVal; } -static void setAllRowsChecked(SLastBlockReader* pLastBlockReader) { -// *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; -} - static bool nextRowInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { -// if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { -// return false; -// } return tMergeTreeNext(&pLastBlockReader->mergeTree); #if 0 From cdeca85e3187f1300371220b8c5daead53a1d263 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Sep 2022 15:19:59 +0800 Subject: [PATCH 070/102] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 36 +++++++++------------ 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index a32721b2b3..a53f67d949 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -189,6 +189,7 @@ bool tLDataIterNextRow(SLDataIter *pIter) { } int32_t iBlockL = pIter->iBlockL; + if (pIter->bData.nRow == 0 && pIter->pBlockL != NULL) { // current block not loaded yet code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pIter->pBlockL, &pIter->bData); if (code != TSDB_CODE_SUCCESS) { @@ -199,30 +200,25 @@ bool tLDataIterNextRow(SLDataIter *pIter) { } pIter->iRow += step; - findNextValidRow(pIter); - if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) { - tLDataIterNextBlock(pIter); - if (pIter->pBlockL == NULL) { // no more data - goto _exit; + while(1) { + findNextValidRow(pIter); + + if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) { + tLDataIterNextBlock(pIter); + if (pIter->pBlockL == NULL) { // no more data + goto _exit; + } + } else { + break; } - } - if (iBlockL != pIter->iBlockL) { - if (pIter->pBlockL) { + if (iBlockL != pIter->iBlockL) { code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pIter->pBlockL, &pIter->bData); if (code) { goto _exit; } - - pIter->iRow = pIter->backward? (pIter->bData.nRow-1):0; - findNextValidRow(pIter); - if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) { - // todo try next block - } - } else { - // no more data - goto _exit; + pIter->iRow = pIter->backward ? (pIter->bData.nRow - 1) : 0; } } @@ -232,10 +228,10 @@ bool tLDataIterNextRow(SLDataIter *pIter) { _exit: if (code != TSDB_CODE_SUCCESS) { - return false; - } else { - return pIter->pBlockL != NULL; + terrno = code; } + + return (code == TSDB_CODE_SUCCESS) && (pIter->pBlockL != NULL); } SRowInfo *tLDataIterGet(SLDataIter *pIter) { From 94f4d746b489c65a3dd0f7b88b4a826a29fc9068 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 1 Sep 2022 15:33:01 +0800 Subject: [PATCH 071/102] tsdbCache: support multi-last data reading --- source/dnode/vnode/src/tsdb/tsdbCache.c | 121 ++++-------------------- 1 file changed, 18 insertions(+), 103 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 059a3a2d83..158cf2a0c3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -418,31 +418,16 @@ typedef enum { } SFSLASTNEXTROWSTATES; typedef struct { - SFSLASTNEXTROWSTATES state; // [input] - STsdb *pTsdb; // [input] - SBlockIdx *pBlockIdxExp; // [input] - STSchema *pTSchema; // [input] - tb_uid_t suid; + SFSLASTNEXTROWSTATES state; // [input] + STsdb *pTsdb; // [input] tb_uid_t uid; int32_t nFileSet; int32_t iFileSet; SArray *aDFileSet; SDataFReader *pDataFReader; - SArray *aBlockL; - SBlockL *pBlockL; - SBlockData *pBlockDataL; - SBlockData blockDataL; - int32_t nRow; - int32_t iRow; TSDBROW row; - /* - SArray *aBlockIdx; - SBlockIdx *pBlockIdx; - SMapData blockMap; - int32_t nBlock; - int32_t iBlock; - SBlock block; - */ + + SMergeTree mergeTree; } SFSLastNextRowIter; static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { @@ -451,22 +436,16 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { switch (state->state) { case SFSLASTNEXTROW_FS: - // state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet; state->nFileSet = taosArrayGetSize(state->aDFileSet); state->iFileSet = state->nFileSet; - state->pBlockDataL = NULL; - case SFSLASTNEXTROW_FILESET: { SDFileSet *pFileSet = NULL; _next_fileset: if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { - if (state->pBlockDataL) { - tBlockDataDestroy(state->pBlockDataL, 1); - state->pBlockDataL = NULL; - } + // tMergeTreeClose(&state->mergeTree); *ppRow = NULL; return code; @@ -475,68 +454,24 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet); if (code) goto _err; - if (!state->aBlockL) { - state->aBlockL = taosArrayInit(0, sizeof(SBlockL)); - } else { - taosArrayClear(state->aBlockL); - } - - code = tsdbReadBlockL(state->pDataFReader, 0, state->aBlockL); - if (code) goto _err; - - // SBlockL *pBlockL = (SBlockL *)taosArrayGet(state->aBlockL, state->iBlockL); - - state->pBlockL = taosArraySearch(state->aBlockL, state->pBlockIdxExp, tCmprBlockL, TD_EQ); - if (!state->pBlockL) { + tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->uid, + &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, + &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}); + bool hasVal = tMergeTreeNext(&state->mergeTree); + if (!hasVal) { + state->state = SFSLASTNEXTROW_FILESET; + // tMergeTreeClose(&state->mergeTree); goto _next_fileset; } - - int64_t suid = state->pBlockL->suid; - int64_t uid = state->pBlockL->maxUid; - - if (!state->pBlockDataL) { - state->pBlockDataL = &state->blockDataL; - - tBlockDataCreate(state->pBlockDataL); - } - code = tBlockDataInit(state->pBlockDataL, suid, suid ? 0 : uid, state->pTSchema); - if (code) goto _err; - } - case SFSLASTNEXTROW_BLOCKDATA: - code = tsdbReadLastBlock(state->pDataFReader, 0, state->pBlockL, state->pBlockDataL); - if (code) goto _err; - - state->nRow = state->blockDataL.nRow; - state->iRow = state->nRow - 1; - - if (!state->pBlockDataL->uid) { - while (state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) { - --state->iRow; - } - } - state->state = SFSLASTNEXTROW_BLOCKROW; + } case SFSLASTNEXTROW_BLOCKROW: - if (state->pBlockDataL->uid) { - if (state->iRow >= 0) { - state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow); - *ppRow = &state->row; - - if (--state->iRow < 0) { - state->state = SFSLASTNEXTROW_FILESET; - } - } - } else { - if (state->iRow >= 0 && state->pBlockIdxExp->uid == state->pBlockDataL->aUid[state->iRow]) { - state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow); - *ppRow = &state->row; - - if (--state->iRow < 0 || state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) { - state->state = SFSLASTNEXTROW_FILESET; - } - } + state->row = tMergeTreeGetRow(&state->mergeTree); + *ppRow = &state->row; + bool hasVal = tMergeTreeNext(&state->mergeTree); + if (!hasVal) { + state->state = SFSLASTNEXTROW_FILESET; } - return code; default: ASSERT(0); @@ -548,15 +483,6 @@ _err: tsdbDataFReaderClose(&state->pDataFReader); state->pDataFReader = NULL; } - if (state->aBlockL) { - taosArrayDestroy(state->aBlockL); - state->aBlockL = NULL; - } - if (state->pBlockDataL) { - tBlockDataDestroy(state->pBlockDataL, 1); - state->pBlockDataL = NULL; - } - *ppRow = NULL; return code; @@ -574,14 +500,6 @@ int32_t clearNextRowFromFSLast(void *iter) { tsdbDataFReaderClose(&state->pDataFReader); state->pDataFReader = NULL; } - if (state->aBlockL) { - taosArrayDestroy(state->aBlockL); - state->aBlockL = NULL; - } - if (state->pBlockDataL) { - tBlockDataDestroy(state->pBlockDataL, 1); - state->pBlockDataL = NULL; - } return code; } @@ -972,9 +890,6 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS; pIter->fsLastState.pTsdb = pTsdb; pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; - pIter->fsLastState.pBlockIdxExp = &pIter->idx; - pIter->fsLastState.pTSchema = pTSchema; - pIter->fsLastState.suid = suid; pIter->fsLastState.uid = uid; pIter->fsState.state = SFSNEXTROW_FS; From bd0523ddb9d8097d772d879daa660eff91abcda1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 1 Sep 2022 15:48:49 +0800 Subject: [PATCH 072/102] tsdbCache/last: add DCLP to last lookup --- source/dnode/vnode/src/tsdb/tsdbCache.c | 42 +++++++++++++++---------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 158cf2a0c3..91eb806940 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1287,25 +1287,33 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand // getTableCacheKeyS(uid, "l", key, &keyLen); getTableCacheKey(uid, 1, key, &keyLen); LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - } else { - SArray *pLastArray = NULL; - code = mergeLast(uid, pTsdb, &pLastArray); - // if table's empty or error, return code of -1 - // if (code < 0 || pRow == NULL) { - if (code < 0 || pLastArray == NULL) { - *handle = NULL; - return 0; - } - - _taos_lru_deleter_t deleter = deleteTableCacheLast; - LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, pLastArray, pLastArray->capacity, deleter, NULL, TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } + if (!h) { + taosThreadMutexLock(&pTsdb->lruMutex); h = taosLRUCacheLookup(pCache, key, keyLen); + if (!h) { + SArray *pLastArray = NULL; + code = mergeLast(uid, pTsdb, &pLastArray); + // if table's empty or error, return code of -1 + // if (code < 0 || pRow == NULL) { + if (code < 0 || pLastArray == NULL) { + *handle = NULL; + return 0; + } + + _taos_lru_deleter_t deleter = deleteTableCacheLast; + LRUStatus status = taosLRUCacheInsert(pCache, key, keyLen, pLastArray, pLastArray->capacity, deleter, NULL, + TAOS_LRU_PRIORITY_LOW); + if (status != TAOS_LRU_STATUS_OK) { + code = -1; + } + + taosThreadMutexUnlock(&pTsdb->lruMutex); + + h = taosLRUCacheLookup(pCache, key, keyLen); + } else { + taosThreadMutexUnlock(&pTsdb->lruMutex); + } } *handle = h; From 72e0f09dd2753165b4a06dcd7f1d40a1969f0020 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Sep 2022 15:50:41 +0800 Subject: [PATCH 073/102] refactor(query): do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 32 +++++++++++++-------- source/dnode/vnode/src/tsdb/tsdbRead.c | 1 + 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index a53f67d949..130a89af5c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -105,12 +105,16 @@ void tLDataIterNextBlock(SLDataIter *pIter) { size_t size = taosArrayGetSize(pIter->aBlockL); for(int32_t i = pIter->iBlockL; i < size && i >= 0; i += step) { SBlockL *p = taosArrayGet(pIter->aBlockL, i); - if (p->minUid <= pIter->uid && p->maxUid >= pIter->uid) { - index = i; + if ((!pIter->backward) && p->minUid > pIter->uid) { break; } - if (p->minUid > pIter->uid) { + if (pIter->backward && p->maxUid < pIter->uid) { + break; + } + + if (p->minUid <= pIter->uid && p->maxUid >= pIter->uid) { + index = i; break; } } @@ -145,8 +149,18 @@ static void findNextValidRow(SLDataIter* pIter) { } int64_t ts = pIter->bData.aTSKEY[i]; - if (ts < pIter->timeWindow.skey) { - continue; + if (!pIter->backward) { // asc + if (ts > pIter->timeWindow.ekey) { // no more data + break; + } else if (ts < pIter->timeWindow.skey) { + continue; + } + } else { + if (ts < pIter->timeWindow.skey) { + break; + } else if (ts > pIter->timeWindow.ekey) { + continue; + } } int64_t ver = pIter->bData.aVersion[i]; @@ -154,11 +168,6 @@ static void findNextValidRow(SLDataIter* pIter) { continue; } - // no data any more, todo opt handle desc case - if (ts > pIter->timeWindow.ekey) { - continue; - } - // todo opt handle desc case if (ver > pIter->verRange.maxVer) { continue; @@ -235,8 +244,7 @@ _exit: } SRowInfo *tLDataIterGet(SLDataIter *pIter) { - // TODO - return NULL; + return &pIter->rInfo; } // SMergeTree ================================================= diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ec0dd8cec4..7ce73adf2a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -636,6 +636,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* { // 1. time range check + printf("%ld, %ld\n", pLastBlock->minKey, pLastBlock->maxKey); if (pLastBlock->minKey > pReader->window.ekey || pLastBlock->maxKey < pReader->window.skey) { continue; } From 97d533d520105ceca51d3a9c4218eb9d7bdfef80 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 1 Sep 2022 16:41:08 +0800 Subject: [PATCH 074/102] tsdbCache/usage: new cacheload column for db.vgroups to check usage --- include/common/tmsg.h | 7 ++++--- source/common/src/systable.c | 1 + source/common/src/tmsg.c | 2 ++ source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndDnode.c | 7 ++++--- source/dnode/mnode/impl/src/mndVgroup.c | 3 +++ source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/tsdb/tsdbCache.c | 2 ++ source/dnode/vnode/src/vnd/vnodeQuery.c | 5 +++-- source/libs/monitor/src/monMsg.c | 5 ++--- 10 files changed, 23 insertions(+), 11 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index c8b90a9991..18770b7b15 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1066,6 +1066,7 @@ typedef struct { typedef struct { int32_t vgId; int32_t syncState; + int64_t cacheUsage; int64_t numOfTables; int64_t numOfTimeSeries; int64_t totalStorage; @@ -2069,9 +2070,9 @@ int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatc // TDMT_VND_DROP_TABLE ================= typedef struct { - char* name; - uint64_t suid; // for tmq in wal format - int8_t igNotExists; + char* name; + uint64_t suid; // for tmq in wal format + int8_t igNotExists; } SVDropTbReq; typedef struct { diff --git a/source/common/src/systable.c b/source/common/src/systable.c index dffef21ac4..23fe56838b 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -206,6 +206,7 @@ static const SSysDbTableSchema vgroupsSchema[] = { {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 1921415239..6337248ab0 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -994,6 +994,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i); if (tEncodeI32(&encoder, pload->vgId) < 0) return -1; if (tEncodeI32(&encoder, pload->syncState) < 0) return -1; + if (tEncodeI64(&encoder, pload->cacheUsage) < 0) return -1; if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1; if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1; if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1; @@ -1063,6 +1064,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { SVnodeLoad vload = {0}; if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1; if (tDecodeI32(&decoder, &vload.syncState) < 0) return -1; + if (tDecodeI64(&decoder, &vload.cacheUsage) < 0) return -1; if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1; if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1; if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index ea05215fe9..624951dadf 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -340,6 +340,7 @@ typedef struct { uint32_t hashEnd; char dbName[TSDB_DB_FNAME_LEN]; int64_t dbUid; + int64_t cacheUsage; int64_t numOfTables; int64_t numOfTimeSeries; int64_t totalStorage; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index fc5e20ef28..26b4080d14 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -347,6 +347,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVload->vgId); if (pVgroup != NULL) { if (pVload->syncState == TAOS_SYNC_STATE_LEADER) { + pVgroup->cacheUsage = pVload->cacheUsage; pVgroup->numOfTables = pVload->numOfTables; pVgroup->numOfTimeSeries = pVload->numOfTimeSeries; pVgroup->totalStorage = pVload->totalStorage; @@ -853,8 +854,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { } int32_t code = -1; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SDnodeObj *pDnode = NULL; pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); @@ -877,7 +878,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { sdbRelease(pSdb, pDnode); } - + if (code == -1) { terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 09eed7fb32..4f02fe2397 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -693,6 +693,9 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppendNULL(pColInfo, numOfRows); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->cacheUsage, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppendNULL(pColInfo, numOfRows); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 5d4285b7c2..f43c0326c0 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -155,6 +155,7 @@ int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int6 void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); size_t tsdbCacheGetCapacity(SVnode *pVnode); +size_t tsdbCacheGetUsage(SVnode *pVnode); // tq typedef struct SMetaTableInfo { diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 91eb806940..a0d52b8b3f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1334,3 +1334,5 @@ void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity) { } size_t tsdbCacheGetCapacity(SVnode *pVnode) { return taosLRUCacheGetCapacity(pVnode->pTsdb->lruCache); } + +size_t tsdbCacheGetUsage(SVnode *pVnode) { return taosLRUCacheGetUsage(pVnode->pTsdb->lruCache); } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 8d799e919d..0d016a2e85 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -368,6 +368,7 @@ _exit: int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->vgId = TD_VID(pVnode); pLoad->syncState = syncGetMyRole(pVnode->sync); + pLoad->cacheUsage = tsdbCacheGetUsage(pVnode); pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta); pLoad->totalStorage = (int64_t)3 * 1073741824; @@ -424,8 +425,8 @@ int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list) { return TSDB_CODE_SUCCESS; } -int32_t vnodeGetStbIdList(SVnode* pVnode, int64_t suid, SArray* list) { - SMStbCursor* pCur = metaOpenStbCursor(pVnode->pMeta, suid); +int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { + SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); if (!pCur) { return TSDB_CODE_FAILED; } diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c index 8fa7e88605..bbee8b1166 100644 --- a/source/libs/monitor/src/monMsg.c +++ b/source/libs/monitor/src/monMsg.c @@ -510,6 +510,7 @@ int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo) SVnodeLoad *pLoad = taosArrayGet(pInfo->pVloads, i); if (tEncodeI32(&encoder, pLoad->vgId) < 0) return -1; if (tEncodeI32(&encoder, pLoad->syncState) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->cacheUsage) < 0) return -1; if (tEncodeI64(&encoder, pLoad->numOfTables) < 0) return -1; if (tEncodeI64(&encoder, pLoad->numOfTimeSeries) < 0) return -1; if (tEncodeI64(&encoder, pLoad->totalStorage) < 0) return -1; @@ -544,6 +545,7 @@ int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInf SVnodeLoad load = {0}; if (tDecodeI32(&decoder, &load.vgId) < 0) return -1; if (tDecodeI32(&decoder, &load.syncState) < 0) return -1; + if (tDecodeI64(&decoder, &load.cacheUsage) < 0) return -1; if (tDecodeI64(&decoder, &load.numOfTables) < 0) return -1; if (tDecodeI64(&decoder, &load.numOfTimeSeries) < 0) return -1; if (tDecodeI64(&decoder, &load.totalStorage) < 0) return -1; @@ -594,7 +596,6 @@ int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInf return 0; } - int32_t tSerializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -639,5 +640,3 @@ int32_t tDeserializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) { tDecoderClear(&decoder); return 0; } - - From a3011a0194147b38a72429c51738e8446800e0cb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Sep 2022 17:51:26 +0800 Subject: [PATCH 075/102] refactor(query): do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 143 +++---------------------- 1 file changed, 16 insertions(+), 127 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7ce73adf2a..9ae4516218 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -34,7 +34,7 @@ typedef struct { typedef struct { int32_t numOfBlocks; - int32_t numOfLastBlocks; + int32_t numOfLastFiles; } SBlockNumber; typedef struct STableBlockScanInfo { @@ -84,7 +84,6 @@ typedef struct SBlockLoadSuppInfo { } SBlockLoadSuppInfo; typedef struct SLastBlockReader { - SArray* pBlockL; int32_t currentBlockIndex; SBlockData lastBlockData; STimeWindow window; @@ -343,7 +342,6 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, } SLastBlockReader* pLReader = pIter->pLastBlockReader; - pLReader->pBlockL = taosArrayInit(4, sizeof(SBlockL)); pLReader->order = pReader->order; pLReader->window = pReader->window; pLReader->verRange = pReader->verRange; @@ -363,6 +361,7 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { int32_t step = asc ? 1 : -1; pIter->index += step; + pIter->pLastBlockReader->uid = 0; if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { return false; } @@ -581,8 +580,7 @@ static void cleanupTableScanInfo(SHashObj* pTableMap) { } } -static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* pLastBlockIndex, - SBlockNumber* pBlockNum, SArray* pQualifiedLastBlock) { +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum) { int32_t numOfQTable = 0; size_t sizeInDisk = 0; size_t numOfTables = taosArrayGetSize(pIndexList); @@ -627,37 +625,14 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* } } - size_t numOfLast = taosArrayGetSize(pLastBlockIndex); - for (int32_t i = 0; i < numOfLast; ++i) { - SBlockL* pLastBlock = taosArrayGet(pLastBlockIndex, i); - if (pLastBlock->suid != pReader->suid) { - continue; - } - - { - // 1. time range check - printf("%ld, %ld\n", pLastBlock->minKey, pLastBlock->maxKey); - if (pLastBlock->minKey > pReader->window.ekey || pLastBlock->maxKey < pReader->window.skey) { - continue; - } - - // 2. version range check - if (pLastBlock->minVer > pReader->verRange.maxVer || pLastBlock->maxVer < pReader->verRange.minVer) { - continue; - } - - pBlockNum->numOfLastBlocks += 1; - taosArrayPush(pQualifiedLastBlock, pLastBlock); - } - } - - int32_t total = pBlockNum->numOfLastBlocks + pBlockNum->numOfBlocks; + pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nLastF; + int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks; double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug( - "load block of %d tables completed, blocks:%d in %d tables, lastBlock:%d, block-info-size:%.2f Kb, elapsed " + "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " "time:%.2f ms %s", - numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastBlocks, sizeInDisk / 1000.0, el, + numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, pReader->idStr); pReader->cost.numOfBlocks += total; @@ -1188,11 +1163,14 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc bool overlapWithDel = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order); // todo here we need to each key in the last files to identify if it is really overlapped with last block + // todo bool overlapWithlastBlock = false; +#if 0 if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0 && (pLastBlockReader->currentBlockIndex != -1)) { SBlockL* pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); } +#endif bool moreThanOutputCapacity = pBlock->nRow > pReader->capacity; bool partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock); @@ -2185,12 +2163,10 @@ static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* p static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { SReaderStatus* pStatus = &pReader->status; pBlockNum->numOfBlocks = 0; - pBlockNum->numOfLastBlocks = 0; + pBlockNum->numOfLastFiles = 0; size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); - SArray* pLastBlocks = pStatus->fileIter.pLastBlockReader->pBlockL; - taosArrayClear(pLastBlocks); while (1) { bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader); @@ -2205,32 +2181,16 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return code; } - code = tsdbReadBlockL(pReader->pFileReader, 0, pLastBlocks); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pIndexList); - return code; - } - - if (taosArrayGetSize(pIndexList) > 0 || taosArrayGetSize(pLastBlocks) > 0) { - SArray* pQLastBlock = taosArrayInit(4, sizeof(SBlockL)); - - code = doLoadFileBlock(pReader, pIndexList, pLastBlocks, pBlockNum, pQLastBlock); + if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nLastF > 0) { + code = doLoadFileBlock(pReader, pIndexList, pBlockNum); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); - taosArrayDestroy(pQLastBlock); return code; } - if (pBlockNum->numOfBlocks + pBlockNum->numOfLastBlocks > 0) { - ASSERT(taosArrayGetSize(pQLastBlock) == pBlockNum->numOfLastBlocks); - taosArrayClear(pLastBlocks); - taosArrayAddAll(pLastBlocks, pQLastBlock); - - taosArrayDestroy(pQLastBlock); + if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) { break; } - - taosArrayDestroy(pQLastBlock); } // no blocks in current file, try next files @@ -2240,69 +2200,6 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return TSDB_CODE_SUCCESS; } -#if 0 -static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo, - STsdbReader* pReader) { - SArray* pBlocks = pLastBlockReader->pBlockL; - SBlockL* pBlock = NULL; - - uint64_t uid = pBlockScanInfo->uid; - int32_t totalLastBlocks = (int32_t)taosArrayGetSize(pBlocks); - - initMemDataIterator(pBlockScanInfo, pReader); - - // find the correct SBlockL. todo binary search - int32_t index = -1; - for (int32_t i = 0; i < totalLastBlocks; ++i) { - SBlockL* p = taosArrayGet(pBlocks, i); - if (p->minUid <= uid && p->maxUid >= uid) { - index = i; - pBlock = p; - break; - } - } - - if (index == -1) { - pLastBlockReader->currentBlockIndex = index; - tBlockDataReset(&pLastBlockReader->lastBlockData); - return TSDB_CODE_SUCCESS; - } - - // the required last datablock has already loaded - if (index == pLastBlockReader->currentBlockIndex) { - return TSDB_CODE_SUCCESS; - } - - int64_t st = taosGetTimestampUs(); - int32_t code = - tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p init block data failed, code:%s %s", pReader, tstrerror(code), pReader->idStr); - return code; - } - - code = tsdbReadLastBlock(pReader->pFileReader, 0, pBlock, &pLastBlockReader->lastBlockData); - - double el = (taosGetTimestampUs() - st) / 1000.0; - if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p error occurs in loading last block into buffer, last block index:%d, total:%d code:%s %s", pReader, - pLastBlockReader->currentBlockIndex, totalLastBlocks, tstrerror(code), pReader->idStr); - } else { - tsdbDebug("%p load last block completed, uid:%" PRIu64 - " last block index:%d, total:%d rows:%d, minVer:%d, maxVer:%d, brange:%" PRId64 "-%" PRId64 - " elapsed time:%.2f ms, %s", - pReader, uid, index, totalLastBlocks, pBlock->nRow, pBlock->minVer, pBlock->maxVer, pBlock->minKey, - pBlock->maxKey, el, pReader->idStr); - } - - pLastBlockReader->currentBlockIndex = index; - pReader->cost.lastBlockLoad += 1; - pReader->cost.lastBlockLoadTime += el; - - return TSDB_CODE_SUCCESS; -} -#endif - static int32_t uidComparFunc(const void* p1, const void* p2) { uint64_t pu1 = *(uint64_t*)p1; uint64_t pu2 = *(uint64_t*)p2; @@ -2395,11 +2292,6 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { while (1) { // load the last data block of current table STableBlockScanInfo* pScanInfo = pStatus->pTableIter; - // code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); - // if (code != TSDB_CODE_SUCCESS) { - // return code; - // } - bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL, pReader->pFileReader); if (!hasVal) { @@ -2566,7 +2458,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl } // all data files are consumed, try data in buffer - if (num.numOfBlocks + num.numOfLastBlocks == 0) { + if (num.numOfBlocks + num.numOfLastFiles == 0) { pReader->status.loadFromFile = false; return code; } @@ -2579,8 +2471,6 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl resetDataBlockIterator(pBlockIter, pReader->order); } - SLastBlockReader* pLReader = pReader->status.fileIter.pLastBlockReader; - // set the correct start position according to the query time window initBlockDumpInfo(pReader, pBlockIter); return code; @@ -2645,7 +2535,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { bool hasNext = blockIteratorNext(&pReader->status.blockIter); if (hasNext) { // check for the next block in the block accessed order list initBlockDumpInfo(pReader, pBlockIter); - } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0) { + } else if (1/*taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0*/) { // data blocks in current file are exhausted, let's try the next file now tBlockDataReset(&pReader->status.fileBlockData); resetDataBlockIterator(pBlockIter, pReader->order); @@ -3444,7 +3334,6 @@ void tsdbReaderClose(STsdbReader* pReader) { SFilesetIter* pFilesetIter = &pReader->status.fileIter; if (pFilesetIter->pLastBlockReader != NULL) { tBlockDataDestroy(&pFilesetIter->pLastBlockReader->lastBlockData, true); - taosArrayDestroy(pFilesetIter->pLastBlockReader->pBlockL); taosMemoryFree(pFilesetIter->pLastBlockReader); } From b2cf2818ebcdf5bd926a5dcb31d6e9e4135a8d22 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Sep 2022 18:23:44 +0800 Subject: [PATCH 076/102] refactor(query): do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 28 +++++--------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9ae4516218..7c720382d6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -84,8 +84,6 @@ typedef struct SBlockLoadSuppInfo { } SBlockLoadSuppInfo; typedef struct SLastBlockReader { - int32_t currentBlockIndex; - SBlockData lastBlockData; STimeWindow window; SVersionRange verRange; int32_t order; @@ -345,11 +343,6 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, pLReader->order = pReader->order; pLReader->window = pReader->window; pLReader->verRange = pReader->verRange; - - int32_t code = tBlockDataCreate(&pLReader->lastBlockData); - if (code != TSDB_CODE_SUCCESS) { - return code; - } } tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); @@ -1269,15 +1262,13 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLast = INT64_MIN; - if ((pLastBlockReader->lastBlockData.nRow > 0) && hasDataInLastBlock(pLastBlockReader)) { + if (hasDataInLastBlock(pLastBlockReader)) { tsLast = getCurrentKeyInLastBlock(pLastBlockReader); } TSDBKEY k = TSDBROW_KEY(pRow); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t minKey = 0; if (pReader->order == TSDB_ORDER_ASC) { minKey = INT64_MAX; // chosen the minimum value @@ -1408,7 +1399,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (pBlockData->nRow > 0) { // no last block available, only data block exists - if (pLastBlockReader->lastBlockData.nRow == 0 || (!hasDataInLastBlock(pLastBlockReader))) { + if (!hasDataInLastBlock(pLastBlockReader)) { return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } @@ -1458,8 +1449,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); ASSERT(pRow != NULL && piRow != NULL); - SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t tsLast = INT64_MIN; + int64_t tsLast = INT64_MIN; if (hasDataInLastBlock(pLastBlockReader)) { tsLast = getCurrentKeyInLastBlock(pLastBlockReader); } @@ -1781,14 +1771,6 @@ static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid } pLastBlockReader->uid = uid; - if (*startPos == -1) { - if (ASCENDING_TRAVERSE(pLastBlockReader->order)) { - // do nothing - } else { - *startPos = pLastBlockReader->lastBlockData.nRow; - } - } - /*int32_t code = */ tMergeTreeOpen(&pLastBlockReader->mergeTree, (pLastBlockReader->order == TSDB_ORDER_DESC), pFReader, uid, &pLastBlockReader->window, &pLastBlockReader->verRange); bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); @@ -2535,7 +2517,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { bool hasNext = blockIteratorNext(&pReader->status.blockIter); if (hasNext) { // check for the next block in the block accessed order list initBlockDumpInfo(pReader, pBlockIter); - } else if (1/*taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0*/) { + } else if (hasDataInLastBlock(pReader->status.fileIter.pLastBlockReader)) { // data blocks in current file are exhausted, let's try the next file now tBlockDataReset(&pReader->status.fileBlockData); resetDataBlockIterator(pBlockIter, pReader->order); @@ -3316,6 +3298,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFreeClear(pSupInfo->buildBuf[i]); } } + taosMemoryFree(pSupInfo->buildBuf); tBlockDataDestroy(&pReader->status.fileBlockData, true); @@ -3333,7 +3316,6 @@ void tsdbReaderClose(STsdbReader* pReader) { SFilesetIter* pFilesetIter = &pReader->status.fileIter; if (pFilesetIter->pLastBlockReader != NULL) { - tBlockDataDestroy(&pFilesetIter->pLastBlockReader->lastBlockData, true); taosMemoryFree(pFilesetIter->pLastBlockReader); } From d1b7d07790d5270aed03a22b8328765a310eea31 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Sep 2022 09:54:22 +0800 Subject: [PATCH 077/102] refactor:do some internal refactor. --- source/dnode/vnode/src/inc/tsdb.h | 10 +---- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 14 ++++++- source/dnode/vnode/src/tsdb/tsdbRead.c | 43 +++++++-------------- source/libs/executor/src/scanoperator.c | 1 + 4 files changed, 29 insertions(+), 39 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index bca180b64b..111c6b5962 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -307,12 +307,6 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode); int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); -struct SLDataIter; -int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iLast, int8_t backward, uint64_t uid, - STimeWindow *pTimeWindow, SVersionRange *pRange); -void tLDataIterClose(struct SLDataIter *pIter); -bool tLDataIterNextRow(struct SLDataIter *pIter); - // structs ======================= struct STsdbFS { SDelFile *pDelFile; @@ -640,16 +634,16 @@ typedef struct { typedef struct SMergeTree { int8_t backward; - SRBTreeNode *pNode; SRBTree rbt; + SArray *pIterList; struct SLDataIter *pIter; - SDataFReader* pLFileReader; } SMergeTree; void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange); void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter); bool tMergeTreeNext(SMergeTree* pMTree); TSDBROW tMergeTreeGetRow(SMergeTree* pMTree); +void tMergeTreeClose(SMergeTree* pMTree); // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 130a89af5c..47c09b51e9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -93,6 +93,7 @@ _exit: void tLDataIterClose(SLDataIter *pIter) { tBlockDataDestroy(&pIter->bData, 1); taosArrayDestroy(pIter->aBlockL); + taosMemoryFree(pIter); } extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData); @@ -272,8 +273,9 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange) { pMTree->backward = backward; - pMTree->pNode = NULL; pMTree->pIter = NULL; + pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); + tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); struct SLDataIter* pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; @@ -281,7 +283,10 @@ void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, /*int32_t code = */tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); bool hasVal = tLDataIterNextRow(pIterList[i]); if (hasVal) { + taosArrayPush(pMTree->pIterList, &pIterList[i]); tMergeTreeAddIter(pMTree, pIterList[i]); + } else { + tLDataIterClose(pIterList[i]); } } } @@ -326,5 +331,12 @@ TSDBROW tMergeTreeGetRow(SMergeTree* pMTree) { } void tMergeTreeClose(SMergeTree* pMTree) { + size_t size = taosArrayGetSize(pMTree->pIterList); + for(int32_t i = 0; i < size; ++i) { + SLDataIter* pIter = taosArrayGetP(pMTree->pIterList, i); + tLDataIterClose(pIter); + } + pMTree->pIterList = taosArrayDestroy(pMTree->pIterList); + pMTree->pIter = NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7c720382d6..bde9aff908 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -48,7 +48,6 @@ typedef struct STableBlockScanInfo { int32_t fileDelIndex; // file block delete index int32_t lastBlockDelIndex; // delete index for last block bool iterInit; // whether to initialize the in-memory skip list iterator or not - int16_t indexInBlockL; // row position in last block } STableBlockScanInfo; typedef struct SBlockOrderWrapper { @@ -228,7 +227,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK } for (int32_t j = 0; j < numOfTables; ++j) { - STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid, .indexInBlockL = INITIAL_ROW_INDEX_VAL}; + STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) { info.lastKey = pTsdbReader->window.skey; @@ -355,6 +354,7 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { pIter->index += step; pIter->pLastBlockReader->uid = 0; + tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { return false; } @@ -567,7 +567,6 @@ static void cleanupTableScanInfo(SHashObj* pTableMap) { } // reset the index in last block when handing a new file - px->indexInBlockL = INITIAL_ROW_INDEX_VAL; tMapDataClear(&px->mapData); taosArrayClear(px->pBlockList); } @@ -1764,17 +1763,20 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, int16_t* startPos, SDataFReader* pFReader) { +static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, SDataFReader* pFReader) { // the last block reader has been initialized for this table. if (pLastBlockReader->uid == uid) { return true; } + if (pLastBlockReader->uid != 0) { + tMergeTreeClose(&pLastBlockReader->mergeTree); + } + pLastBlockReader->uid = uid; /*int32_t code = */ tMergeTreeOpen(&pLastBlockReader->mergeTree, (pLastBlockReader->order == TSDB_ORDER_DESC), pFReader, uid, &pLastBlockReader->window, &pLastBlockReader->verRange); - bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); - return hasVal; + return tMergeTreeNext(&pLastBlockReader->mergeTree); } static bool nextRowInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { @@ -2274,8 +2276,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { while (1) { // load the last data block of current table STableBlockScanInfo* pScanInfo = pStatus->pTableIter; - bool hasVal = - initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL, pReader->pFileReader); + bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo->uid, pReader->pFileReader); if (!hasVal) { bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); if (!hasNexTable) { @@ -2284,26 +2285,6 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { continue; } - // int32_t index = pScanInfo->indexInBlockL; - - // if (index == INITIAL_ROW_INDEX_VAL || index == pLastBlockReader->lastBlockData.nRow) { - // bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); - // if (!hasData) { // current table does not have rows in last block, try next table - // bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); - // if (!hasNexTable) { - // return TSDB_CODE_SUCCESS; - // } - // continue; - // } - // } - // } else { // no data in last block, try next table - // bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); - // if (!hasNexTable) { - // return TSDB_CODE_SUCCESS; - // } - // continue; - // } - code = doBuildDataBlock(pReader); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3144,7 +3125,7 @@ int32_t tsdbSetTableId(STsdbReader* pReader, int64_t uid) { ASSERT(pReader != NULL); taosHashClear(pReader->status.pTableMap); - STableBlockScanInfo info = {.lastKey = 0, .uid = uid, .indexInBlockL = INITIAL_ROW_INDEX_VAL}; + STableBlockScanInfo info = {.lastKey = 0, .uid = uid}; taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); return TDB_CODE_SUCCESS; } @@ -3287,7 +3268,6 @@ void tsdbReaderClose(STsdbReader* pReader) { } SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap); taosMemoryFreeClear(pSupInfo->plist); taosMemoryFree(pSupInfo->colIds); @@ -3312,10 +3292,13 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbDataFReaderClose(&pReader->pFileReader); } + tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap); + taosMemoryFree(pReader->status.uidCheckInfo.tableUidList); SFilesetIter* pFilesetIter = &pReader->status.fileIter; if (pFilesetIter->pLastBlockReader != NULL) { + tMergeTreeClose(&pFilesetIter->pLastBlockReader->mergeTree); taosMemoryFree(pFilesetIter->pLastBlockReader); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b740ec21d3..47c04991c0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -695,6 +695,7 @@ static void destroyTableScanOperatorInfo(void* param) { cleanupQueryTableDataCond(&pTableScanInfo->cond); tsdbReaderClose(pTableScanInfo->dataReader); + pTableScanInfo->dataReader = NULL; if (pTableScanInfo->pColMatchInfo != NULL) { taosArrayDestroy(pTableScanInfo->pColMatchInfo); From 3286ab2a9eaf023c65f26d5da48dccd6dd65d750 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 10:09:38 +0800 Subject: [PATCH 078/102] refact --- source/common/src/tglobal.c | 4 ---- source/dnode/vnode/src/inc/tsdb.h | 23 +++++++++-------------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 908f7c014e..c436e4ffd2 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -129,10 +129,6 @@ int32_t tsMinIntervalTime = 1; int32_t tsQueryBufferSize = -1; int64_t tsQueryBufferSizeBytes = -1; -// tsdb config -// For backward compatibility -bool tsdbForceKeepFile = false; - int32_t tsDiskCfgNum = 0; SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 111c6b5962..28e82d5be0 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -478,12 +478,6 @@ struct SBlockData { SArray *aColData; // SArray }; -// ================== TSDB global config -extern bool tsdbForceKeepFile; - -#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC -#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC - struct TABLEID { tb_uid_t suid; tb_uid_t uid; @@ -633,17 +627,18 @@ typedef struct { } SRowInfo; typedef struct SMergeTree { - int8_t backward; - SRBTree rbt; - SArray *pIterList; - struct SLDataIter *pIter; + int8_t backward; + SRBTree rbt; + SArray *pIterList; + struct SLDataIter *pIter; } SMergeTree; -void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange); +void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t uid, STimeWindow *pTimeWindow, + SVersionRange *pVerRange); void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter); -bool tMergeTreeNext(SMergeTree* pMTree); -TSDBROW tMergeTreeGetRow(SMergeTree* pMTree); -void tMergeTreeClose(SMergeTree* pMTree); +bool tMergeTreeNext(SMergeTree *pMTree); +TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); +void tMergeTreeClose(SMergeTree *pMTree); // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { From 1132a7f71924a31c43f32d55c0c25ee6755e3c34 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 10:17:40 +0800 Subject: [PATCH 079/102] refact code --- source/dnode/vnode/src/inc/tsdb.h | 14 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 28 +-- source/dnode/vnode/src/tsdb/tsdbFS.c | 168 +++++++++--------- source/dnode/vnode/src/tsdb/tsdbFile.c | 38 ++-- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 51 +++--- source/dnode/vnode/src/tsdb/tsdbRead.c | 26 ++- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 52 +++--- source/dnode/vnode/src/tsdb/tsdbRetention.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 12 +- 9 files changed, 193 insertions(+), 198 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 28e82d5be0..f24abfabfc 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -50,7 +50,7 @@ typedef struct SBlockData SBlockData; typedef struct SDelFile SDelFile; typedef struct SHeadFile SHeadFile; typedef struct SDataFile SDataFile; -typedef struct SLastFile SLastFile; +typedef struct SSstFile SSstFile; typedef struct SSmaFile SSmaFile; typedef struct SDFileSet SDFileSet; typedef struct SDataFWriter SDataFWriter; @@ -219,7 +219,7 @@ bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype); int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile); int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile); -int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile); +int32_t tPutSstFile(uint8_t *p, SSstFile *pSstFile); int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile); int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile); int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile); @@ -228,7 +228,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet); void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]); void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]); -void tsdbLastFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SLastFile *pLastF, char fname[]); +void tsdbSstFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSstFile *pSstF, char fname[]); void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]); // SDelFile void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]); @@ -541,7 +541,7 @@ struct SDataFile { int64_t size; }; -struct SLastFile { +struct SSstFile { volatile int32_t nRef; int64_t commitID; @@ -562,8 +562,8 @@ struct SDFileSet { SHeadFile *pHeadF; SDataFile *pDataF; SSmaFile *pSmaF; - uint8_t nLastF; - SLastFile *aLastF[TSDB_MAX_LAST_FILE]; + uint8_t nSstF; + SSstFile *aSstF[TSDB_MAX_LAST_FILE]; }; struct SRowIter { @@ -598,7 +598,7 @@ struct SDataFWriter { SHeadFile fHead; SDataFile fData; SSmaFile fSma; - SLastFile fLast[TSDB_MAX_LAST_FILE]; + SSstFile fSst[TSDB_MAX_LAST_FILE]; uint8_t *aBuf[4]; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3c497d143f..50671eb695 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -431,9 +431,9 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { pCommitter->toLastOnly = 0; SDataFReader *pReader = pCommitter->dReader.pReader; if (pReader) { - if (pReader->pSet->nLastF >= pCommitter->maxLast) { + if (pReader->pSet->nSstF >= pCommitter->maxLast) { int8_t iIter = 0; - for (int32_t iLast = 0; iLast < pReader->pSet->nLastF; iLast++) { + for (int32_t iLast = 0; iLast < pReader->pSet->nSstF; iLast++) { pIter = &pCommitter->aDataIter[iIter]; pIter->type = LAST_DATA_ITER; pIter->iLast = iLast; @@ -457,9 +457,9 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { iIter++; } } else { - for (int32_t iLast = 0; iLast < pReader->pSet->nLastF; iLast++) { - SLastFile *pLastFile = pReader->pSet->aLastF[iLast]; - if (pLastFile->size > pLastFile->offset) { + for (int32_t iLast = 0; iLast < pReader->pSet->nSstF; iLast++) { + SSstFile *pSstFile = pReader->pSet->aSstF[iLast]; + if (pSstFile->size > pSstFile->offset) { pCommitter->toLastOnly = 1; break; } @@ -515,29 +515,29 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { SHeadFile fHead = {.commitID = pCommitter->commitID}; SDataFile fData = {.commitID = pCommitter->commitID}; SSmaFile fSma = {.commitID = pCommitter->commitID}; - SLastFile fLast = {.commitID = pCommitter->commitID}; + SSstFile fSst = {.commitID = pCommitter->commitID}; SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma}; if (pRSet) { - ASSERT(pRSet->nLastF <= pCommitter->maxLast); + ASSERT(pRSet->nSstF <= pCommitter->maxLast); fData = *pRSet->pDataF; fSma = *pRSet->pSmaF; wSet.diskId = pRSet->diskId; - if (pRSet->nLastF < pCommitter->maxLast) { - for (int32_t iLast = 0; iLast < pRSet->nLastF; iLast++) { - wSet.aLastF[iLast] = pRSet->aLastF[iLast]; + if (pRSet->nSstF < pCommitter->maxLast) { + for (int32_t iLast = 0; iLast < pRSet->nSstF; iLast++) { + wSet.aSstF[iLast] = pRSet->aSstF[iLast]; } - wSet.nLastF = pRSet->nLastF + 1; + wSet.nSstF = pRSet->nSstF + 1; } else { - wSet.nLastF = 1; + wSet.nSstF = 1; } } else { SDiskID did = {0}; tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did); tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); wSet.diskId = did; - wSet.nLastF = 1; + wSet.nSstF = 1; } - wSet.aLastF[wSet.nLastF - 1] = &fLast; + wSet.aSstF[wSet.nSstF - 1] = &fSst; code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 1a142fde1a..775bd3be52 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -255,8 +255,8 @@ void tsdbFSDestroy(STsdbFS *pFS) { taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pDataF); taosMemoryFree(pSet->pSmaF); - for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { - taosMemoryFree(pSet->aLastF[iLast]); + for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { + taosMemoryFree(pSet->aSstF[iLast]); } } @@ -312,12 +312,12 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { } // last =========== - tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[0], fname); + tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[0], fname); if (taosStatFile(fname, &size, NULL)) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size != pSet->aLastF[0]->size) { + if (size != pSet->aSstF[0]->size) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -509,8 +509,8 @@ int32_t tsdbFSClose(STsdb *pTsdb) { taosMemoryFree(pSet->pDataF); // last - ASSERT(pSet->aLastF[0]->nRef == 1); - taosMemoryFree(pSet->aLastF[0]); + ASSERT(pSet->aSstF[0]->nRef == 1); + taosMemoryFree(pSet->aSstF[0]); // sma ASSERT(pSet->pSmaF->nRef == 1); @@ -571,13 +571,13 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { *fSet.pSmaF = *pSet->pSmaF; // last - for (fSet.nLastF = 0; fSet.nLastF < pSet->nLastF; fSet.nLastF++) { - fSet.aLastF[fSet.nLastF] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.aLastF[fSet.nLastF] == NULL) { + for (fSet.nSstF = 0; fSet.nSstF < pSet->nSstF; fSet.nSstF++) { + fSet.aSstF[fSet.nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (fSet.aSstF[fSet.nSstF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - *fSet.aLastF[fSet.nLastF] = *pSet->aLastF[fSet.nLastF]; + *fSet.aSstF[fSet.nSstF] = *pSet->aSstF[fSet.nSstF]; } if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { @@ -631,27 +631,27 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { *pDFileSet->pDataF = *pSet->pDataF; *pDFileSet->pSmaF = *pSet->pSmaF; // last - if (pSet->nLastF > pDFileSet->nLastF) { - ASSERT(pSet->nLastF == pDFileSet->nLastF + 1); + if (pSet->nSstF > pDFileSet->nSstF) { + ASSERT(pSet->nSstF == pDFileSet->nSstF + 1); - pDFileSet->aLastF[pDFileSet->nLastF] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (pDFileSet->aLastF[pDFileSet->nLastF] == NULL) { + pDFileSet->aSstF[pDFileSet->nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (pDFileSet->aSstF[pDFileSet->nSstF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - *pDFileSet->aLastF[pDFileSet->nLastF] = *pSet->aLastF[pSet->nLastF - 1]; - pDFileSet->nLastF++; - } else if (pSet->nLastF < pDFileSet->nLastF) { - ASSERT(pSet->nLastF == 1); - for (int32_t iLast = 1; iLast < pDFileSet->nLastF; iLast++) { - taosMemoryFree(pDFileSet->aLastF[iLast]); + *pDFileSet->aSstF[pDFileSet->nSstF] = *pSet->aSstF[pSet->nSstF - 1]; + pDFileSet->nSstF++; + } else if (pSet->nSstF < pDFileSet->nSstF) { + ASSERT(pSet->nSstF == 1); + for (int32_t iLast = 1; iLast < pDFileSet->nSstF; iLast++) { + taosMemoryFree(pDFileSet->aSstF[iLast]); } - *pDFileSet->aLastF[0] = *pSet->aLastF[0]; - pDFileSet->nLastF = 1; + *pDFileSet->aSstF[0] = *pSet->aSstF[0]; + pDFileSet->nSstF = 1; } else { - for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { - *pDFileSet->aLastF[iLast] = *pSet->aLastF[iLast]; + for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { + *pDFileSet->aSstF[iLast] = *pSet->aSstF[iLast]; } } @@ -659,8 +659,8 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } } - ASSERT(pSet->nLastF == 1); - SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nLastF = 1}; + ASSERT(pSet->nSstF == 1); + SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nSstF = 1}; // head fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); @@ -687,12 +687,12 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { *fSet.pSmaF = *pSet->pSmaF; // last - fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.aLastF[0] == NULL) { + fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (fSet.aSstF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - *fSet.aLastF[0] = *pSet->aLastF[0]; + *fSet.aSstF[0] = *pSet->aSstF[0]; if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -862,74 +862,74 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { // last if (sameDisk) { - if (pSetNew->nLastF > pSetOld->nLastF) { - ASSERT(pSetNew->nLastF = pSetOld->nLastF + 1); - pSetOld->aLastF[pSetOld->nLastF] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (pSetOld->aLastF[pSetOld->nLastF] == NULL) { + if (pSetNew->nSstF > pSetOld->nSstF) { + ASSERT(pSetNew->nSstF = pSetOld->nSstF + 1); + pSetOld->aSstF[pSetOld->nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (pSetOld->aSstF[pSetOld->nSstF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *pSetOld->aLastF[pSetOld->nLastF] = *pSetNew->aLastF[pSetOld->nLastF]; - pSetOld->aLastF[pSetOld->nLastF]->nRef = 1; - pSetOld->nLastF++; - } else if (pSetNew->nLastF < pSetOld->nLastF) { - ASSERT(pSetNew->nLastF == 1); - for (int32_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { - SLastFile *pLastFile = pSetOld->aLastF[iLast]; - nRef = atomic_sub_fetch_32(&pLastFile->nRef, 1); + *pSetOld->aSstF[pSetOld->nSstF] = *pSetNew->aSstF[pSetOld->nSstF]; + pSetOld->aSstF[pSetOld->nSstF]->nRef = 1; + pSetOld->nSstF++; + } else if (pSetNew->nSstF < pSetOld->nSstF) { + ASSERT(pSetNew->nSstF == 1); + for (int32_t iLast = 0; iLast < pSetOld->nSstF; iLast++) { + SSstFile *pSstFile = pSetOld->aSstF[iLast]; + nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); if (nRef == 0) { - tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pLastFile, fname); + tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); taosRemoveFile(fname); - taosMemoryFree(pLastFile); + taosMemoryFree(pSstFile); } - pSetOld->aLastF[iLast] = NULL; + pSetOld->aSstF[iLast] = NULL; } - pSetOld->nLastF = 1; - pSetOld->aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (pSetOld->aLastF[0] == NULL) { + pSetOld->nSstF = 1; + pSetOld->aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (pSetOld->aSstF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *pSetOld->aLastF[0] = *pSetNew->aLastF[0]; - pSetOld->aLastF[0]->nRef = 1; + *pSetOld->aSstF[0] = *pSetNew->aSstF[0]; + pSetOld->aSstF[0]->nRef = 1; } else { - for (int32_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { - SLastFile *pLastFile = pSetOld->aLastF[iLast]; - nRef = atomic_sub_fetch_32(&pLastFile->nRef, 1); + for (int32_t iLast = 0; iLast < pSetOld->nSstF; iLast++) { + SSstFile *pSstFile = pSetOld->aSstF[iLast]; + nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); if (nRef == 0) { - tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pLastFile, fname); + tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); taosRemoveFile(fname); - taosMemoryFree(pLastFile); + taosMemoryFree(pSstFile); } - pSetOld->aLastF[iLast] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (pSetOld->aLastF[iLast] == NULL) { + pSetOld->aSstF[iLast] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (pSetOld->aSstF[iLast] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *pSetOld->aLastF[iLast] = *pSetNew->aLastF[iLast]; - pSetOld->aLastF[iLast]->nRef = 1; + *pSetOld->aSstF[iLast] = *pSetNew->aSstF[iLast]; + pSetOld->aSstF[iLast]->nRef = 1; } } } else { - ASSERT(pSetOld->nLastF == pSetNew->nLastF); - for (int32_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { - SLastFile *pLastFile = pSetOld->aLastF[iLast]; - nRef = atomic_sub_fetch_32(&pLastFile->nRef, 1); + ASSERT(pSetOld->nSstF == pSetNew->nSstF); + for (int32_t iLast = 0; iLast < pSetOld->nSstF; iLast++) { + SSstFile *pSstFile = pSetOld->aSstF[iLast]; + nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); if (nRef == 0) { - tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pLastFile, fname); + tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); taosRemoveFile(fname); - taosMemoryFree(pLastFile); + taosMemoryFree(pSstFile); } - pSetOld->aLastF[iLast] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (pSetOld->aLastF[iLast] == NULL) { + pSetOld->aSstF[iLast] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (pSetOld->aSstF[iLast] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *pSetOld->aLastF[iLast] = *pSetNew->aLastF[iLast]; - pSetOld->aLastF[iLast]->nRef = 1; + *pSetOld->aSstF[iLast] = *pSetNew->aSstF[iLast]; + pSetOld->aSstF[iLast]->nRef = 1; } } @@ -963,12 +963,12 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { taosMemoryFree(pSetOld->pSmaF); } - for (int8_t iLast = 0; iLast < pSetOld->nLastF; iLast++) { - nRef = atomic_sub_fetch_32(&pSetOld->aLastF[iLast]->nRef, 1); + for (int8_t iLast = 0; iLast < pSetOld->nSstF; iLast++) { + nRef = atomic_sub_fetch_32(&pSetOld->aSstF[iLast]->nRef, 1); if (nRef == 0) { - tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aLastF[iLast], fname); + tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSstF[iLast], fname); taosRemoveFile(fname); - taosMemoryFree(pSetOld->aLastF[iLast]); + taosMemoryFree(pSetOld->aSstF[iLast]); } } @@ -976,7 +976,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { continue; _add_new: - fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nLastF = 1}; + fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSstF = 1}; // head fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); @@ -1006,14 +1006,14 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { fSet.pSmaF->nRef = 1; // last - ASSERT(pSetNew->nLastF == 1); - fSet.aLastF[0] = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.aLastF[0] == NULL) { + ASSERT(pSetNew->nSstF == 1); + fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (fSet.aSstF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *fSet.aLastF[0] = *pSetNew->aLastF[0]; - fSet.aLastF[0]->nRef = 1; + *fSet.aSstF[0] = *pSetNew->aSstF[0]; + fSet.aSstF[0]->nRef = 1; if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1061,8 +1061,8 @@ int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1); ASSERT(nRef > 0); - for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { - nRef = atomic_fetch_add_32(&pSet->aLastF[iLast]->nRef, 1); + for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { + nRef = atomic_fetch_add_32(&pSet->aSstF[iLast]->nRef, 1); ASSERT(nRef > 0); } @@ -1121,13 +1121,13 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { } // last - for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { - nRef = atomic_sub_fetch_32(&pSet->aLastF[iLast]->nRef, 1); + for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { + nRef = atomic_sub_fetch_32(&pSet->aSstF[iLast]->nRef, 1); ASSERT(nRef >= 0); if (nRef == 0) { - tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[iLast], fname); + tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iLast], fname); taosRemoveFile(fname); - taosMemoryFree(pSet->aLastF[iLast]); + taosMemoryFree(pSet->aSstF[iLast]); /* code */ } } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 4814d793f7..01562de8be 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -53,22 +53,22 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) { return n; } -int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile) { +int32_t tPutSstFile(uint8_t *p, SSstFile *pSstFile) { int32_t n = 0; - n += tPutI64v(p ? p + n : p, pLastFile->commitID); - n += tPutI64v(p ? p + n : p, pLastFile->size); - n += tPutI64v(p ? p + n : p, pLastFile->offset); + n += tPutI64v(p ? p + n : p, pSstFile->commitID); + n += tPutI64v(p ? p + n : p, pSstFile->size); + n += tPutI64v(p ? p + n : p, pSstFile->offset); return n; } -static int32_t tGetLastFile(uint8_t *p, SLastFile *pLastFile) { +static int32_t tGetSstFile(uint8_t *p, SSstFile *pSstFile) { int32_t n = 0; - n += tGetI64v(p + n, &pLastFile->commitID); - n += tGetI64v(p + n, &pLastFile->size); - n += tGetI64v(p + n, &pLastFile->offset); + n += tGetI64v(p + n, &pSstFile->commitID); + n += tGetI64v(p + n, &pSstFile->size); + n += tGetI64v(p + n, &pSstFile->offset); return n; } @@ -102,9 +102,9 @@ void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pDataF->commitID, ".data"); } -void tsdbLastFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SLastFile *pLastF, char fname[]) { +void tsdbSstFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSstFile *pSstF, char fname[]) { snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did), - TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pLastF->commitID, ".last"); + TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pSstF->commitID, ".sst"); } void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) { @@ -195,9 +195,9 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { n += tPutSmaFile(p ? p + n : p, pSet->pSmaF); // last - n += tPutU8(p ? p + n : p, pSet->nLastF); - for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { - n += tPutLastFile(p ? p + n : p, pSet->aLastF[iLast]); + n += tPutU8(p ? p + n : p, pSet->nSstF); + for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { + n += tPutSstFile(p ? p + n : p, pSet->aSstF[iLast]); } return n; @@ -235,14 +235,14 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { n += tGetSmaFile(p + n, pSet->pSmaF); // last - n += tGetU8(p + n, &pSet->nLastF); - for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { - pSet->aLastF[iLast] = (SLastFile *)taosMemoryCalloc(1, sizeof(SLastFile)); - if (pSet->aLastF[iLast] == NULL) { + n += tGetU8(p + n, &pSet->nSstF); + for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { + pSet->aSstF[iLast] = (SSstFile *)taosMemoryCalloc(1, sizeof(SSstFile)); + if (pSet->aSstF[iLast] == NULL) { return -1; } - pSet->aLastF[iLast]->nRef = 1; - n += tGetLastFile(p + n, pSet->aLastF[iLast]); + pSet->aSstF[iLast]->nRef = 1; + n += tGetSstFile(p + n, pSet->aSstF[iLast]); } return n; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 47c09b51e9..527f92d2a9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -99,12 +99,12 @@ void tLDataIterClose(SLDataIter *pIter) { extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData); void tLDataIterNextBlock(SLDataIter *pIter) { - int32_t step = pIter->backward? -1:1; + int32_t step = pIter->backward ? -1 : 1; pIter->iBlockL += step; int32_t index = -1; - size_t size = taosArrayGetSize(pIter->aBlockL); - for(int32_t i = pIter->iBlockL; i < size && i >= 0; i += step) { + size_t size = taosArrayGetSize(pIter->aBlockL); + for (int32_t i = pIter->iBlockL; i < size && i >= 0; i += step) { SBlockL *p = taosArrayGet(pIter->aBlockL, i); if ((!pIter->backward) && p->minUid > pIter->uid) { break; @@ -122,15 +122,15 @@ void tLDataIterNextBlock(SLDataIter *pIter) { if (index == -1) { pIter->pBlockL = NULL; - } else { + } else { pIter->pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); } } -static void findNextValidRow(SLDataIter* pIter) { - int32_t step = pIter->backward? -1:1; +static void findNextValidRow(SLDataIter *pIter) { + int32_t step = pIter->backward ? -1 : 1; - bool hasVal = false; + bool hasVal = false; int32_t i = pIter->iRow; for (; i < pIter->bData.nRow && i >= 0; i += step) { if (pIter->bData.aUid != NULL) { @@ -150,7 +150,7 @@ static void findNextValidRow(SLDataIter* pIter) { } int64_t ts = pIter->bData.aTSKEY[i]; - if (!pIter->backward) { // asc + if (!pIter->backward) { // asc if (ts > pIter->timeWindow.ekey) { // no more data break; } else if (ts < pIter->timeWindow.skey) { @@ -186,12 +186,12 @@ static void findNextValidRow(SLDataIter* pIter) { break; } - pIter->iRow = (hasVal)? i:-1; + pIter->iRow = (hasVal) ? i : -1; } bool tLDataIterNextRow(SLDataIter *pIter) { int32_t code = 0; - int32_t step = pIter->backward? -1:1; + int32_t step = pIter->backward ? -1 : 1; // no qualified last file block in current file, no need to fetch row if (pIter->pBlockL == NULL) { @@ -206,12 +206,12 @@ bool tLDataIterNextRow(SLDataIter *pIter) { goto _exit; } - pIter->iRow = (pIter->backward)? pIter->bData.nRow:-1; + pIter->iRow = (pIter->backward) ? pIter->bData.nRow : -1; } pIter->iRow += step; - while(1) { + while (1) { findNextValidRow(pIter); if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) { @@ -237,16 +237,14 @@ bool tLDataIterNextRow(SLDataIter *pIter) { pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); _exit: - if (code != TSDB_CODE_SUCCESS) { + if (code != TSDB_CODE_SUCCESS) { terrno = code; } return (code == TSDB_CODE_SUCCESS) && (pIter->pBlockL != NULL); } -SRowInfo *tLDataIterGet(SLDataIter *pIter) { - return &pIter->rInfo; -} +SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } // SMergeTree ================================================= static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { @@ -271,16 +269,17 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { } } -void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange) { +void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t uid, STimeWindow *pTimeWindow, + SVersionRange *pVerRange) { pMTree->backward = backward; pMTree->pIter = NULL; pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); - struct SLDataIter* pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; - for(int32_t i = 0; i < pFReader->pSet->nLastF; ++i) { // open all last file - /*int32_t code = */tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); + struct SLDataIter *pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; + for (int32_t i = 0; i < pFReader->pSet->nSstF; ++i) { // open all last file + /*int32_t code = */ tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); bool hasVal = tLDataIterNextRow(pIterList[i]); if (hasVal) { taosArrayPush(pMTree->pIterList, &pIterList[i]); @@ -293,7 +292,7 @@ void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } -bool tMergeTreeNext(SMergeTree* pMTree) { +bool tMergeTreeNext(SMergeTree *pMTree) { int32_t code = TSDB_CODE_SUCCESS; if (pMTree->pIter) { SLDataIter *pIter = pMTree->pIter; @@ -326,14 +325,12 @@ bool tMergeTreeNext(SMergeTree* pMTree) { return pMTree->pIter != NULL; } -TSDBROW tMergeTreeGetRow(SMergeTree* pMTree) { - return pMTree->pIter->rInfo.row; -} +TSDBROW tMergeTreeGetRow(SMergeTree *pMTree) { return pMTree->pIter->rInfo.row; } -void tMergeTreeClose(SMergeTree* pMTree) { +void tMergeTreeClose(SMergeTree *pMTree) { size_t size = taosArrayGetSize(pMTree->pIterList); - for(int32_t i = 0; i < size; ++i) { - SLDataIter* pIter = taosArrayGetP(pMTree->pIterList, i); + for (int32_t i = 0; i < size; ++i) { + SLDataIter *pIter = taosArrayGetP(pMTree->pIterList, i); tLDataIterClose(pIter); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2d70cc2e4d..697498a2bb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -176,10 +176,10 @@ static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pR static void setComposedBlockFlag(STsdbReader* pReader, bool composed); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); -static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, - STsdbReader* pReader, bool* freeTSRow); -static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, - STSRow** pTSRow); +static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, + STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow); +static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader, STSRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader); @@ -617,7 +617,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN } } - pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nLastF; + pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nSstF; int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks; double el = (taosGetTimestampUs() - st) / 1000.0; @@ -1374,7 +1374,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, bool mergeBlockData) { - int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); STSRow* pTSRow = NULL; SRowMerger merge = {0}; @@ -1860,9 +1860,7 @@ static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { return key.ts; } -static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { - return pLastBlockReader->mergeTree.pIter != NULL; -} +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader) { @@ -1958,7 +1956,6 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } } - bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); // no data in last block and block, no need to proceed. @@ -2185,7 +2182,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return code; } - if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nLastF > 0) { + if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nSstF > 0) { code = doLoadFileBlock(pReader, pIndexList, pBlockNum); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); @@ -2296,7 +2293,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { while (1) { // load the last data block of current table STableBlockScanInfo* pScanInfo = pStatus->pTableIter; - bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo->uid, pReader->pFileReader); + bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo->uid, pReader->pFileReader); if (!hasVal) { bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); if (!hasNexTable) { @@ -2946,7 +2943,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, } int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, - STSRow** pTSRow) { + STSRow** pTSRow) { SRowMerger merge = {0}; TSDBKEY k = TSDBROW_KEY(pRow); @@ -3020,7 +3017,8 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR } if (pBlockScanInfo->iter.hasVal && pRow != NULL) { - return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); + return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, + freeTSRow); } if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 664c7d1572..989c9e3e2c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -434,8 +434,8 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS } // last - for (int32_t iLast = 0; iLast < pSet->nLastF; iLast++) { - tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aLastF[iLast], fname); + for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { + tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iLast], fname); pReader->aLastFD[iLast] = taosOpenFile(fname, TD_FILE_READ); if (pReader->aLastFD[iLast] == NULL) { code = TAOS_SYSTEM_ERROR(errno); @@ -475,7 +475,7 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { } // last - for (int32_t iLast = 0; iLast < (*ppReader)->pSet->nLastF; iLast++) { + for (int32_t iLast = 0; iLast < (*ppReader)->pSet->nSstF; iLast++) { if (taosCloseFile(&(*ppReader)->aLastFD[iLast]) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -561,8 +561,8 @@ _err: int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL) { int32_t code = 0; - int64_t offset = pReader->pSet->aLastF[iLast]->offset; - int64_t size = pReader->pSet->aLastF[iLast]->size - offset; + int64_t offset = pReader->pSet->aSstF[iLast]->offset; + int64_t size = pReader->pSet->aSstF[iLast]->size - offset; int64_t n; uint32_t delimiter; @@ -947,14 +947,14 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS .pHeadF = &pWriter->fHead, .pDataF = &pWriter->fData, .pSmaF = &pWriter->fSma, - .nLastF = pSet->nLastF // + .nSstF = pSet->nSstF // }; pWriter->fHead = *pSet->pHeadF; pWriter->fData = *pSet->pDataF; pWriter->fSma = *pSet->pSmaF; - for (int8_t iLast = 0; iLast < pSet->nLastF; iLast++) { - pWriter->wSet.aLastF[iLast] = &pWriter->fLast[iLast]; - pWriter->fLast[iLast] = *pSet->aLastF[iLast]; + for (int8_t iLast = 0; iLast < pSet->nSstF; iLast++) { + pWriter->wSet.aSstF[iLast] = &pWriter->fSst[iLast]; + pWriter->fSst[iLast] = *pSet->aSstF[iLast]; } // head @@ -1037,9 +1037,9 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS } // last - ASSERT(pWriter->fLast[pSet->nLastF - 1].size == 0); + ASSERT(pWriter->fSst[pSet->nSstF - 1].size == 0); flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - tsdbLastFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fLast[pSet->nLastF - 1], fname); + tsdbSstFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSst[pSet->nSstF - 1], fname); pWriter->pLastFD = taosOpenFile(fname, flag); if (pWriter->pLastFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); @@ -1050,7 +1050,7 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS code = TAOS_SYSTEM_ERROR(errno); goto _err; } - pWriter->fLast[pWriter->wSet.nLastF - 1].size += TSDB_FHDR_SIZE; + pWriter->fSst[pWriter->wSet.nSstF - 1].size += TSDB_FHDR_SIZE; *ppWriter = pWriter; return code; @@ -1181,7 +1181,7 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { // last ============== memset(hdr, 0, TSDB_FHDR_SIZE); - tPutLastFile(hdr, &pWriter->fLast[pWriter->wSet.nLastF - 1]); + tPutSstFile(hdr, &pWriter->fSst[pWriter->wSet.nSstF - 1]); taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET); @@ -1302,14 +1302,14 @@ _err: } int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { - int32_t code = 0; - SLastFile *pLastFile = &pWriter->fLast[pWriter->wSet.nLastF - 1]; - int64_t size; - int64_t n; + int32_t code = 0; + SSstFile *pSstFile = &pWriter->fSst[pWriter->wSet.nSstF - 1]; + int64_t size; + int64_t n; // check if (taosArrayGetSize(aBlockL) == 0) { - pLastFile->offset = pLastFile->size; + pSstFile->offset = pSstFile->size; goto _exit; } @@ -1342,12 +1342,12 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { } // update - pLastFile->offset = pLastFile->size; - pLastFile->size += size; + pSstFile->offset = pSstFile->size; + pSstFile->size += size; _exit: tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), - pLastFile->offset, size); + pSstFile->offset, size); return code; _err: @@ -1431,7 +1431,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock ASSERT(pBlockData->nRow > 0); - pBlkInfo->offset = toLast ? pWriter->fLast[pWriter->wSet.nLastF - 1].size : pWriter->fData.size; + pBlkInfo->offset = toLast ? pWriter->fSst[pWriter->wSet.nSstF - 1].size : pWriter->fData.size; pBlkInfo->szBlock = 0; pBlkInfo->szKey = 0; @@ -1475,7 +1475,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock // update info if (toLast) { - pWriter->fLast[pWriter->wSet.nLastF - 1].size += pBlkInfo->szBlock; + pWriter->fSst[pWriter->wSet.nSstF - 1].size += pBlkInfo->szBlock; } else { pWriter->fData.size += pBlkInfo->szBlock; } @@ -1555,8 +1555,8 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { taosCloseFile(&PInFD); // last - tsdbLastFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aLastF[0], fNameFrom); - tsdbLastFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aLastF[0], fNameTo); + tsdbSstFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSstF[0], fNameFrom); + tsdbSstFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSstF[0], fNameTo); pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { @@ -1570,7 +1570,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { goto _err; } - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aLastF[0]->size); + n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aSstF[0]->size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 9fd2c4f6cd..ee29538a81 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -60,7 +60,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { if (expLevel < 0) { taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->aLastF[0]); + taosMemoryFree(pSet->aSstF[0]); taosMemoryFree(pSet->pSmaF); taosArrayRemove(fs.aDFileSet, iSet); iSet--; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index fa7d370583..cd265aa174 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -931,25 +931,25 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 // write SHeadFile fHead; SDataFile fData; - SLastFile fLast; + SSstFile fLast; SSmaFile fSma; - SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aLastF[0] = &fLast, .pSmaF = &fSma}; + SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aSstF[0] = &fLast, .pSmaF = &fSma}; if (pSet) { wSet.diskId = pSet->diskId; wSet.fid = fid; - wSet.nLastF = 1; + wSet.nSstF = 1; fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fData = *pSet->pDataF; - fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; + fLast = (SSstFile){.commitID = pWriter->commitID, .size = 0}; fSma = *pSet->pSmaF; } else { wSet.diskId = (SDiskID){.level = 0, .id = 0}; wSet.fid = fid; - wSet.nLastF = 1; + wSet.nSstF = 1; fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fData = (SDataFile){.commitID = pWriter->commitID, .size = 0}; - fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0, .offset = 0}; + fLast = (SSstFile){.commitID = pWriter->commitID, .size = 0, .offset = 0}; fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0}; } From 7d23c83091adf2f9a838f971948a3853bbe8adba Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 10:22:38 +0800 Subject: [PATCH 080/102] refact code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 28 ++++++++++--------- source/dnode/vnode/src/tsdb/tsdbFile.c | 4 +-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 10 +++---- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 50671eb695..f27b8c7576 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -37,7 +37,7 @@ typedef struct { int32_t iBlockL; SBlockData bData; int32_t iRow; - }; // last file data iter + }; // sst file data iter }; } SDataIter; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 775bd3be52..8f6ab96379 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -110,7 +110,7 @@ _err: // taosRemoveFile(fname); // } -// // last +// // sst // if (isSameDisk && pFrom->pLastF->commitID == pTo->pLastF->commitID) { // if (pFrom->pLastF->size > pTo->pLastF->size) { // code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_LAST_FILE); @@ -140,7 +140,7 @@ _err: // tsdbDataFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pDataF, fname); // taosRemoveFile(fname); -// // last +// // sst // tsdbLastFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pLastF, fname); // taosRemoveFile(fname); @@ -311,7 +311,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { if (code) goto _err; } - // last =========== + // sst =========== tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[0], fname); if (taosStatFile(fname, &size, NULL)) { code = TAOS_SYSTEM_ERROR(errno); @@ -508,13 +508,15 @@ int32_t tsdbFSClose(STsdb *pTsdb) { ASSERT(pSet->pDataF->nRef == 1); taosMemoryFree(pSet->pDataF); - // last - ASSERT(pSet->aSstF[0]->nRef == 1); - taosMemoryFree(pSet->aSstF[0]); - // sma ASSERT(pSet->pSmaF->nRef == 1); taosMemoryFree(pSet->pSmaF); + + // sst + for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { + ASSERT(pSet->aSstF[iSst]->nRef == 1); + taosMemoryFree(pSet->aSstF[iSst]); + } } taosArrayDestroy(pTsdb->fs.aDFileSet); @@ -570,7 +572,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { } *fSet.pSmaF = *pSet->pSmaF; - // last + // sst for (fSet.nSstF = 0; fSet.nSstF < pSet->nSstF; fSet.nSstF++) { fSet.aSstF[fSet.nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); if (fSet.aSstF[fSet.nSstF] == NULL) { @@ -630,7 +632,7 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { *pDFileSet->pHeadF = *pSet->pHeadF; *pDFileSet->pDataF = *pSet->pDataF; *pDFileSet->pSmaF = *pSet->pSmaF; - // last + // sst if (pSet->nSstF > pDFileSet->nSstF) { ASSERT(pSet->nSstF == pDFileSet->nSstF + 1); @@ -686,7 +688,7 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } *fSet.pSmaF = *pSet->pSmaF; - // last + // sst fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); if (fSet.aSstF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -860,7 +862,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { pSetOld->pSmaF->size = pSetNew->pSmaF->size; } - // last + // sst if (sameDisk) { if (pSetNew->nSstF > pSetOld->nSstF) { ASSERT(pSetNew->nSstF = pSetOld->nSstF + 1); @@ -1005,7 +1007,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { *fSet.pSmaF = *pSetNew->pSmaF; fSet.pSmaF->nRef = 1; - // last + // sst ASSERT(pSetNew->nSstF == 1); fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); if (fSet.aSstF[0] == NULL) { @@ -1120,7 +1122,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { taosMemoryFree(pSet->pSmaF); } - // last + // sst for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { nRef = atomic_sub_fetch_32(&pSet->aSstF[iLast]->nRef, 1); ASSERT(nRef >= 0); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 01562de8be..573021ecd6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -194,7 +194,7 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { n += tPutDataFile(p ? p + n : p, pSet->pDataF); n += tPutSmaFile(p ? p + n : p, pSet->pSmaF); - // last + // sst n += tPutU8(p ? p + n : p, pSet->nSstF); for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { n += tPutSstFile(p ? p + n : p, pSet->aSstF[iLast]); @@ -234,7 +234,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { pSet->pSmaF->nRef = 1; n += tGetSmaFile(p + n, pSet->pSmaF); - // last + // sst n += tGetU8(p + n, &pSet->nSstF); for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { pSet->aSstF[iLast] = (SSstFile *)taosMemoryCalloc(1, sizeof(SSstFile)); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 989c9e3e2c..b0e798b96d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -433,7 +433,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS goto _err; } - // last + // sst for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iLast], fname); pReader->aLastFD[iLast] = taosOpenFile(fname, TD_FILE_READ); @@ -474,7 +474,7 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { goto _err; } - // last + // sst for (int32_t iLast = 0; iLast < (*ppReader)->pSet->nSstF; iLast++) { if (taosCloseFile(&(*ppReader)->aLastFD[iLast]) < 0) { code = TAOS_SYSTEM_ERROR(errno); @@ -1036,7 +1036,7 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS ASSERT(n == pWriter->fSma.size); } - // last + // sst ASSERT(pWriter->fSst[pSet->nSstF - 1].size == 0); flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; tsdbSstFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSst[pSet->nSstF - 1], fname); @@ -1179,7 +1179,7 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { goto _err; } - // last ============== + // sst ============== memset(hdr, 0, TSDB_FHDR_SIZE); tPutSstFile(hdr, &pWriter->fSst[pWriter->wSet.nSstF - 1]); taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); @@ -1554,7 +1554,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { taosCloseFile(&pOutFD); taosCloseFile(&PInFD); - // last + // sst tsdbSstFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSstF[0], fNameFrom); tsdbSstFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSstF[0], fNameTo); From ff4fda84421c0f23e629488c62b5e7369cf21d01 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 10:43:49 +0800 Subject: [PATCH 081/102] refact code --- source/dnode/vnode/src/inc/tsdb.h | 16 ++-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 74 +++++++++---------- source/dnode/vnode/src/tsdb/tsdbFS.c | 62 ++++++++-------- source/dnode/vnode/src/tsdb/tsdbFile.c | 14 ++-- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 60 +++++++-------- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 58 +++++++-------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 52 ++++++------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 56 +++++++------- 9 files changed, 199 insertions(+), 199 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f24abfabfc..5a9d6c43af 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -43,7 +43,7 @@ typedef struct STbDataIter STbDataIter; typedef struct SMapData SMapData; typedef struct SBlockIdx SBlockIdx; typedef struct SBlock SBlock; -typedef struct SBlockL SBlockL; +typedef struct SSstBlk SSstBlk; typedef struct SColData SColData; typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SBlockData SBlockData; @@ -120,9 +120,9 @@ int32_t tPutBlock(uint8_t *p, void *ph); int32_t tGetBlock(uint8_t *p, void *ph); int32_t tBlockCmprFn(const void *p1, const void *p2); bool tBlockHasSma(SBlock *pBlock); -// SBlockL -int32_t tPutBlockL(uint8_t *p, void *ph); -int32_t tGetBlockL(uint8_t *p, void *ph); +// SSstBlk +int32_t tPutSstBlk(uint8_t *p, void *ph); +int32_t tGetSstBlk(uint8_t *p, void *ph); // SBlockIdx int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph); @@ -254,7 +254,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx); -int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL); +int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast); @@ -264,10 +264,10 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); -int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL); +int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk); int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData); -int32_t tsdbReadLastBlock(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData); +int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData); // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); @@ -439,7 +439,7 @@ struct SBlock { SSmaInfo smaInfo; }; -struct SBlockL { +struct SSstBlk { int64_t suid; int64_t minUid; int64_t maxUid; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index f27b8c7576..caab533f6e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -32,9 +32,9 @@ typedef struct { STbDataIter iter; }; // memory data iter struct { - int32_t iLast; - SArray *aBlockL; - int32_t iBlockL; + int32_t iSst; + SArray *aSstBlk; + int32_t iSstBlk; SBlockData bData; int32_t iRow; }; // sst file data iter @@ -77,7 +77,7 @@ typedef struct { struct { SDataFWriter *pWriter; SArray *aBlockIdx; // SArray - SArray *aBlockL; // SArray + SArray *aSstBlk; // SArray SMapData mBlock; // SMapData SBlockData bData; SBlockData bDatal; @@ -92,8 +92,8 @@ typedef struct { SArray *aDelData; // SArray } SCommitter; -extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, - SBlockData *pBlockData); // todo +extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *aSstBlk, + SBlockData *pBlockData); // todo static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter); static int32_t tsdbCommitData(SCommitter *pCommitter); @@ -433,19 +433,19 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { if (pReader) { if (pReader->pSet->nSstF >= pCommitter->maxLast) { int8_t iIter = 0; - for (int32_t iLast = 0; iLast < pReader->pSet->nSstF; iLast++) { + for (int32_t iSst = 0; iSst < pReader->pSet->nSstF; iSst++) { pIter = &pCommitter->aDataIter[iIter]; pIter->type = LAST_DATA_ITER; - pIter->iLast = iLast; + pIter->iSst = iSst; - code = tsdbReadBlockL(pCommitter->dReader.pReader, iLast, pIter->aBlockL); + code = tsdbReadSstBlk(pCommitter->dReader.pReader, iSst, pIter->aSstBlk); if (code) goto _err; - if (taosArrayGetSize(pIter->aBlockL) == 0) continue; + if (taosArrayGetSize(pIter->aSstBlk) == 0) continue; - pIter->iBlockL = 0; - SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, 0); - code = tsdbReadLastBlockEx(pCommitter->dReader.pReader, iLast, pBlockL, &pIter->bData); + pIter->iSstBlk = 0; + SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, 0); + code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, iSst, pSstBlk, &pIter->bData); if (code) goto _err; pIter->iRow = 0; @@ -457,8 +457,8 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { iIter++; } } else { - for (int32_t iLast = 0; iLast < pReader->pSet->nSstF; iLast++) { - SSstFile *pSstFile = pReader->pSet->aSstF[iLast]; + for (int32_t iSst = 0; iSst < pReader->pSet->nSstF; iSst++) { + SSstFile *pSstFile = pReader->pSet->aSstF[iSst]; if (pSstFile->size > pSstFile->offset) { pCommitter->toLastOnly = 1; break; @@ -523,8 +523,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { fSma = *pRSet->pSmaF; wSet.diskId = pRSet->diskId; if (pRSet->nSstF < pCommitter->maxLast) { - for (int32_t iLast = 0; iLast < pRSet->nSstF; iLast++) { - wSet.aSstF[iLast] = pRSet->aSstF[iLast]; + for (int32_t iSst = 0; iSst < pRSet->nSstF; iSst++) { + wSet.aSstF[iSst] = pRSet->aSstF[iSst]; } wSet.nSstF = pRSet->nSstF + 1; } else { @@ -542,7 +542,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { if (code) goto _err; taosArrayClear(pCommitter->dWriter.aBlockIdx); - taosArrayClear(pCommitter->dWriter.aBlockL); + taosArrayClear(pCommitter->dWriter.aSstBlk); tMapDataReset(&pCommitter->dWriter.mBlock); tBlockDataReset(&pCommitter->dWriter.bData); tBlockDataReset(&pCommitter->dWriter.bDatal); @@ -613,7 +613,7 @@ _err: static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { int32_t code = 0; - SBlockL blockL; + SSstBlk blockL; SBlockData *pBlockData = &pCommitter->dWriter.bDatal; ASSERT(pBlockData->nRow > 0); @@ -638,8 +638,8 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1); if (code) goto _err; - // push SBlockL - if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { + // push SSstBlk + if (taosArrayPush(pCommitter->dWriter.aSstBlk, &blockL) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -661,8 +661,8 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); if (code) goto _err; - // write aBlockL - code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL); + // write aSstBlk + code = tsdbWriteSstBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSstBlk); if (code) goto _err; // update file header @@ -790,10 +790,10 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { if (code) goto _exit; // merger - for (int32_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { - SDataIter *pIter = &pCommitter->aDataIter[iLast]; - pIter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pIter->aBlockL == NULL) { + for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) { + SDataIter *pIter = &pCommitter->aDataIter[iSst]; + pIter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk)); + if (pIter->aSstBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -809,8 +809,8 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { goto _exit; } - pCommitter->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pCommitter->dWriter.aBlockL == NULL) { + pCommitter->dWriter.aSstBlk = taosArrayInit(0, sizeof(SSstBlk)); + if (pCommitter->dWriter.aSstBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -832,15 +832,15 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { tBlockDataDestroy(&pCommitter->dReader.bData, 1); // merger - for (int32_t iLast = 0; iLast < TSDB_MAX_LAST_FILE; iLast++) { - SDataIter *pIter = &pCommitter->aDataIter[iLast]; - taosArrayDestroy(pIter->aBlockL); + for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) { + SDataIter *pIter = &pCommitter->aDataIter[iSst]; + taosArrayDestroy(pIter->aSstBlk); tBlockDataDestroy(&pIter->bData, 1); } // writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); - taosArrayDestroy(pCommitter->dWriter.aBlockL); + taosArrayDestroy(pCommitter->dWriter.aSstBlk); tMapDataClear(&pCommitter->dWriter.mBlock); tBlockDataDestroy(&pCommitter->dWriter.bData, 1); tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); @@ -1055,11 +1055,11 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; pIter->r.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); } else { - pIter->iBlockL++; - if (pIter->iBlockL < taosArrayGetSize(pIter->aBlockL)) { - SBlockL *pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); + pIter->iSstBlk++; + if (pIter->iSstBlk < taosArrayGetSize(pIter->aSstBlk)) { + SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk); - code = tsdbReadLastBlockEx(pCommitter->dReader.pReader, pIter->iLast, pBlockL, &pIter->bData); + code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, pIter->iSst, pSstBlk, &pIter->bData); if (code) goto _exit; pIter->iRow = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 8f6ab96379..37bd1ccbf6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -255,8 +255,8 @@ void tsdbFSDestroy(STsdbFS *pFS) { taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pDataF); taosMemoryFree(pSet->pSmaF); - for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { - taosMemoryFree(pSet->aSstF[iLast]); + for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { + taosMemoryFree(pSet->aSstF[iSst]); } } @@ -645,15 +645,15 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { pDFileSet->nSstF++; } else if (pSet->nSstF < pDFileSet->nSstF) { ASSERT(pSet->nSstF == 1); - for (int32_t iLast = 1; iLast < pDFileSet->nSstF; iLast++) { - taosMemoryFree(pDFileSet->aSstF[iLast]); + for (int32_t iSst = 1; iSst < pDFileSet->nSstF; iSst++) { + taosMemoryFree(pDFileSet->aSstF[iSst]); } *pDFileSet->aSstF[0] = *pSet->aSstF[0]; pDFileSet->nSstF = 1; } else { - for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { - *pDFileSet->aSstF[iLast] = *pSet->aSstF[iLast]; + for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { + *pDFileSet->aSstF[iSst] = *pSet->aSstF[iSst]; } } @@ -876,15 +876,15 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { pSetOld->nSstF++; } else if (pSetNew->nSstF < pSetOld->nSstF) { ASSERT(pSetNew->nSstF == 1); - for (int32_t iLast = 0; iLast < pSetOld->nSstF; iLast++) { - SSstFile *pSstFile = pSetOld->aSstF[iLast]; + for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) { + SSstFile *pSstFile = pSetOld->aSstF[iSst]; nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); if (nRef == 0) { tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); taosRemoveFile(fname); taosMemoryFree(pSstFile); } - pSetOld->aSstF[iLast] = NULL; + pSetOld->aSstF[iSst] = NULL; } pSetOld->nSstF = 1; @@ -896,8 +896,8 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { *pSetOld->aSstF[0] = *pSetNew->aSstF[0]; pSetOld->aSstF[0]->nRef = 1; } else { - for (int32_t iLast = 0; iLast < pSetOld->nSstF; iLast++) { - SSstFile *pSstFile = pSetOld->aSstF[iLast]; + for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) { + SSstFile *pSstFile = pSetOld->aSstF[iSst]; nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); if (nRef == 0) { tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); @@ -905,19 +905,19 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { taosMemoryFree(pSstFile); } - pSetOld->aSstF[iLast] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (pSetOld->aSstF[iLast] == NULL) { + pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (pSetOld->aSstF[iSst] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *pSetOld->aSstF[iLast] = *pSetNew->aSstF[iLast]; - pSetOld->aSstF[iLast]->nRef = 1; + *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst]; + pSetOld->aSstF[iSst]->nRef = 1; } } } else { ASSERT(pSetOld->nSstF == pSetNew->nSstF); - for (int32_t iLast = 0; iLast < pSetOld->nSstF; iLast++) { - SSstFile *pSstFile = pSetOld->aSstF[iLast]; + for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) { + SSstFile *pSstFile = pSetOld->aSstF[iSst]; nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); if (nRef == 0) { tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); @@ -925,13 +925,13 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { taosMemoryFree(pSstFile); } - pSetOld->aSstF[iLast] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (pSetOld->aSstF[iLast] == NULL) { + pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (pSetOld->aSstF[iSst] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - *pSetOld->aSstF[iLast] = *pSetNew->aSstF[iLast]; - pSetOld->aSstF[iLast]->nRef = 1; + *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst]; + pSetOld->aSstF[iSst]->nRef = 1; } } @@ -965,12 +965,12 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { taosMemoryFree(pSetOld->pSmaF); } - for (int8_t iLast = 0; iLast < pSetOld->nSstF; iLast++) { - nRef = atomic_sub_fetch_32(&pSetOld->aSstF[iLast]->nRef, 1); + for (int8_t iSst = 0; iSst < pSetOld->nSstF; iSst++) { + nRef = atomic_sub_fetch_32(&pSetOld->aSstF[iSst]->nRef, 1); if (nRef == 0) { - tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSstF[iLast], fname); + tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSstF[iSst], fname); taosRemoveFile(fname); - taosMemoryFree(pSetOld->aSstF[iLast]); + taosMemoryFree(pSetOld->aSstF[iSst]); } } @@ -1063,8 +1063,8 @@ int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1); ASSERT(nRef > 0); - for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { - nRef = atomic_fetch_add_32(&pSet->aSstF[iLast]->nRef, 1); + for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { + nRef = atomic_fetch_add_32(&pSet->aSstF[iSst]->nRef, 1); ASSERT(nRef > 0); } @@ -1123,13 +1123,13 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { } // sst - for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { - nRef = atomic_sub_fetch_32(&pSet->aSstF[iLast]->nRef, 1); + for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { + nRef = atomic_sub_fetch_32(&pSet->aSstF[iSst]->nRef, 1); ASSERT(nRef >= 0); if (nRef == 0) { - tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iLast], fname); + tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname); taosRemoveFile(fname); - taosMemoryFree(pSet->aSstF[iLast]); + taosMemoryFree(pSet->aSstF[iSst]); /* code */ } } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 573021ecd6..632a2c827b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -196,8 +196,8 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { // sst n += tPutU8(p ? p + n : p, pSet->nSstF); - for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { - n += tPutSstFile(p ? p + n : p, pSet->aSstF[iLast]); + for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { + n += tPutSstFile(p ? p + n : p, pSet->aSstF[iSst]); } return n; @@ -236,13 +236,13 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { // sst n += tGetU8(p + n, &pSet->nSstF); - for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { - pSet->aSstF[iLast] = (SSstFile *)taosMemoryCalloc(1, sizeof(SSstFile)); - if (pSet->aSstF[iLast] == NULL) { + for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { + pSet->aSstF[iSst] = (SSstFile *)taosMemoryCalloc(1, sizeof(SSstFile)); + if (pSet->aSstF[iSst] == NULL) { return -1; } - pSet->aSstF[iLast]->nRef = 1; - n += tGetSstFile(p + n, pSet->aSstF[iLast]); + pSet->aSstF[iSst]->nRef = 1; + n += tGetSstFile(p + n, pSet->aSstF[iSst]); } return n; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 527f92d2a9..8d38b34679 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -18,12 +18,12 @@ // SLDataIter ================================================= typedef struct SLDataIter { SRBTreeNode node; - SBlockL *pBlockL; + SSstBlk *pSstBlk; SDataFReader *pReader; - int32_t iLast; + int32_t iSst; int8_t backward; - SArray *aBlockL; - int32_t iBlockL; + SArray *aSstBlk; + int32_t iSstBlk; SBlockData bData; int32_t iRow; SRowInfo rInfo; @@ -32,7 +32,7 @@ typedef struct SLDataIter { SVersionRange verRange; } SLDataIter; -int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iLast, int8_t backward, uint64_t uid, +int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iSst, int8_t backward, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange) { int32_t code = 0; *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); @@ -41,10 +41,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t (*pIter)->timeWindow = *pTimeWindow; (*pIter)->verRange = *pRange; (*pIter)->pReader = pReader; - (*pIter)->iLast = iLast; + (*pIter)->iSst = iSst; (*pIter)->backward = backward; - (*pIter)->aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if ((*pIter)->aBlockL == NULL) { + (*pIter)->aSstBlk = taosArrayInit(0, sizeof(SSstBlk)); + if ((*pIter)->aSstBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -54,18 +54,18 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t goto _exit; } - code = tsdbReadBlockL(pReader, iLast, (*pIter)->aBlockL); + code = tsdbReadSstBlk(pReader, iSst, (*pIter)->aSstBlk); if (code) { goto _exit; } - size_t size = taosArrayGetSize((*pIter)->aBlockL); + size_t size = taosArrayGetSize((*pIter)->aSstBlk); // find the start block int32_t index = -1; if (!backward) { // asc for (int32_t i = 0; i < size; ++i) { - SBlockL *p = taosArrayGet((*pIter)->aBlockL, i); + SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i); if (p->minUid <= uid && p->maxUid >= uid) { index = i; break; @@ -73,7 +73,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t } } else { // desc for (int32_t i = size - 1; i >= 0; --i) { - SBlockL *p = taosArrayGet((*pIter)->aBlockL, i); + SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i); if (p->minUid <= uid && p->maxUid >= uid) { index = i; break; @@ -81,9 +81,9 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t } } - (*pIter)->iBlockL = index; + (*pIter)->iSstBlk = index; if (index != -1) { - (*pIter)->pBlockL = taosArrayGet((*pIter)->aBlockL, (*pIter)->iBlockL); + (*pIter)->pSstBlk = taosArrayGet((*pIter)->aSstBlk, (*pIter)->iSstBlk); } _exit: @@ -92,20 +92,20 @@ _exit: void tLDataIterClose(SLDataIter *pIter) { tBlockDataDestroy(&pIter->bData, 1); - taosArrayDestroy(pIter->aBlockL); + taosArrayDestroy(pIter->aSstBlk); taosMemoryFree(pIter); } -extern int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData); +extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData); void tLDataIterNextBlock(SLDataIter *pIter) { int32_t step = pIter->backward ? -1 : 1; - pIter->iBlockL += step; + pIter->iSstBlk += step; int32_t index = -1; - size_t size = taosArrayGetSize(pIter->aBlockL); - for (int32_t i = pIter->iBlockL; i < size && i >= 0; i += step) { - SBlockL *p = taosArrayGet(pIter->aBlockL, i); + size_t size = taosArrayGetSize(pIter->aSstBlk); + for (int32_t i = pIter->iSstBlk; i < size && i >= 0; i += step) { + SSstBlk *p = taosArrayGet(pIter->aSstBlk, i); if ((!pIter->backward) && p->minUid > pIter->uid) { break; } @@ -121,9 +121,9 @@ void tLDataIterNextBlock(SLDataIter *pIter) { } if (index == -1) { - pIter->pBlockL = NULL; + pIter->pSstBlk = NULL; } else { - pIter->pBlockL = (SBlockL *)taosArrayGet(pIter->aBlockL, pIter->iBlockL); + pIter->pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk); } } @@ -194,14 +194,14 @@ bool tLDataIterNextRow(SLDataIter *pIter) { int32_t step = pIter->backward ? -1 : 1; // no qualified last file block in current file, no need to fetch row - if (pIter->pBlockL == NULL) { + if (pIter->pSstBlk == NULL) { return false; } - int32_t iBlockL = pIter->iBlockL; + int32_t iBlockL = pIter->iSstBlk; - if (pIter->bData.nRow == 0 && pIter->pBlockL != NULL) { // current block not loaded yet - code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pIter->pBlockL, &pIter->bData); + if (pIter->bData.nRow == 0 && pIter->pSstBlk != NULL) { // current block not loaded yet + code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, &pIter->bData); if (code != TSDB_CODE_SUCCESS) { goto _exit; } @@ -216,15 +216,15 @@ bool tLDataIterNextRow(SLDataIter *pIter) { if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) { tLDataIterNextBlock(pIter); - if (pIter->pBlockL == NULL) { // no more data + if (pIter->pSstBlk == NULL) { // no more data goto _exit; } } else { break; } - if (iBlockL != pIter->iBlockL) { - code = tsdbReadLastBlockEx(pIter->pReader, pIter->iLast, pIter->pBlockL, &pIter->bData); + if (iBlockL != pIter->iSstBlk) { + code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, &pIter->bData); if (code) { goto _exit; } @@ -241,7 +241,7 @@ _exit: terrno = code; } - return (code == TSDB_CODE_SUCCESS) && (pIter->pBlockL != NULL); + return (code == TSDB_CODE_SUCCESS) && (pIter->pSstBlk != NULL); } SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 697498a2bb..ef0c23fd58 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1158,9 +1158,9 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc // todo bool overlapWithlastBlock = false; #if 0 - if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0 && (pLastBlockReader->currentBlockIndex != -1)) { - SBlockL* pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); - overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); + if (taosArrayGetSize(pLastBlockReader->pSstBlk) > 0 && (pLastBlockReader->currentBlockIndex != -1)) { + SSstBlk* pSstBlk = taosArrayGet(pLastBlockReader->pSstBlk, pLastBlockReader->currentBlockIndex); + overlapWithlastBlock = !(pBlock->maxKey.ts < pSstBlk->minKey || pBlock->minKey.ts > pSstBlk->maxKey); } #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index b0e798b96d..d9c85f55e0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -434,10 +434,10 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS } // sst - for (int32_t iLast = 0; iLast < pSet->nSstF; iLast++) { - tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iLast], fname); - pReader->aLastFD[iLast] = taosOpenFile(fname, TD_FILE_READ); - if (pReader->aLastFD[iLast] == NULL) { + for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { + tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname); + pReader->aLastFD[iSst] = taosOpenFile(fname, TD_FILE_READ); + if (pReader->aLastFD[iSst] == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -475,8 +475,8 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { } // sst - for (int32_t iLast = 0; iLast < (*ppReader)->pSet->nSstF; iLast++) { - if (taosCloseFile(&(*ppReader)->aLastFD[iLast]) < 0) { + for (int32_t iSst = 0; iSst < (*ppReader)->pSet->nSstF; iSst++) { + if (taosCloseFile(&(*ppReader)->aLastFD[iSst]) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -559,14 +559,14 @@ _err: return code; } -int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL) { +int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) { int32_t code = 0; - int64_t offset = pReader->pSet->aSstF[iLast]->offset; - int64_t size = pReader->pSet->aSstF[iLast]->size - offset; + int64_t offset = pReader->pSet->aSstF[iSst]->offset; + int64_t size = pReader->pSet->aSstF[iSst]->size - offset; int64_t n; uint32_t delimiter; - taosArrayClear(aBlockL); + taosArrayClear(aSstBlk); if (size == 0) { goto _exit; } @@ -576,13 +576,13 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL) { if (code) goto _err; // seek - if (taosLSeekFile(pReader->aLastFD[iLast], offset, SEEK_SET) < 0) { + if (taosLSeekFile(pReader->aLastFD[iSst], offset, SEEK_SET) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } // read - n = taosReadFile(pReader->aLastFD[iLast], pReader->aBuf[0], size); + n = taosReadFile(pReader->aLastFD[iSst], pReader->aBuf[0], size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -603,10 +603,10 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, int32_t iLast, SArray *aBlockL) { ASSERT(delimiter == TSDB_FILE_DLMT); while (n < size - sizeof(TSCKSUM)) { - SBlockL blockl; - n += tGetBlockL(pReader->aBuf[0] + n, &blockl); + SSstBlk blockl; + n += tGetSstBlk(pReader->aBuf[0] + n, &blockl); - if (taosArrayPush(aBlockL, &blockl) == NULL) { + if (taosArrayPush(aSstBlk, &blockl) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -897,10 +897,10 @@ _err: return code; } -int32_t tsdbReadLastBlock(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData) { +int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) { int32_t code = 0; - code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, pBlockData); + code = tsdbReadBlockDataImpl(pReader, &pSstBlk->bInfo, 1, pBlockData); if (code) goto _err; return code; @@ -910,15 +910,15 @@ _err: return code; } -int32_t tsdbReadLastBlockEx(SDataFReader *pReader, int32_t iLast, SBlockL *pBlockL, SBlockData *pBlockData) { +int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) { int32_t code = 0; // read - code = tsdbReadAndCheck(pReader->aLastFD[iLast], pBlockL->bInfo.offset, &pReader->aBuf[0], pBlockL->bInfo.szBlock, 0); + code = tsdbReadAndCheck(pReader->aLastFD[iSst], pSstBlk->bInfo.offset, &pReader->aBuf[0], pSstBlk->bInfo.szBlock, 0); if (code) goto _exit; // decmpr - code = tDecmprBlockData(pReader->aBuf[0], pBlockL->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); + code = tDecmprBlockData(pReader->aBuf[0], pSstBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); if (code) goto _exit; _exit: @@ -952,9 +952,9 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS pWriter->fHead = *pSet->pHeadF; pWriter->fData = *pSet->pDataF; pWriter->fSma = *pSet->pSmaF; - for (int8_t iLast = 0; iLast < pSet->nSstF; iLast++) { - pWriter->wSet.aSstF[iLast] = &pWriter->fSst[iLast]; - pWriter->fSst[iLast] = *pSet->aSstF[iLast]; + for (int8_t iSst = 0; iSst < pSet->nSstF; iSst++) { + pWriter->wSet.aSstF[iSst] = &pWriter->fSst[iSst]; + pWriter->fSst[iSst] = *pSet->aSstF[iSst]; } // head @@ -1301,22 +1301,22 @@ _err: return code; } -int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { +int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) { int32_t code = 0; SSstFile *pSstFile = &pWriter->fSst[pWriter->wSet.nSstF - 1]; int64_t size; int64_t n; // check - if (taosArrayGetSize(aBlockL) == 0) { + if (taosArrayGetSize(aSstBlk) == 0) { pSstFile->offset = pSstFile->size; goto _exit; } // size size = sizeof(uint32_t); // TSDB_FILE_DLMT - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { - size += tPutBlockL(NULL, taosArrayGet(aBlockL, iBlockL)); + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) { + size += tPutSstBlk(NULL, taosArrayGet(aSstBlk, iBlockL)); } size += sizeof(TSCKSUM); @@ -1327,8 +1327,8 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { // encode n = 0; n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { - n += tPutBlockL(pWriter->aBuf[0] + n, taosArrayGet(aBlockL, iBlockL)); + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) { + n += tPutSstBlk(pWriter->aBuf[0] + n, taosArrayGet(aSstBlk, iBlockL)); } taosCalcChecksumAppend(0, pWriter->aBuf[0], size); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index cd265aa174..62c279ef06 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -27,9 +27,9 @@ struct STsdbSnapReader { int32_t fid; SDataFReader* pDataFReader; SArray* aBlockIdx; // SArray - SArray* aBlockL; // SArray + SArray* aSstBlk; // SArray SBlockIdx* pBlockIdx; - SBlockL* pBlockL; + SSstBlk* pSstBlk; int32_t iBlockIdx; int32_t iBlockL; @@ -64,7 +64,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); if (code) goto _err; - code = tsdbReadBlockL(pReader->pDataFReader, 0, pReader->aBlockL); + code = tsdbReadSstBlk(pReader->pDataFReader, 0, pReader->aSstBlk); if (code) goto _err; // init @@ -82,13 +82,13 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->iBlockL = 0; while (true) { - if (pReader->iBlockL >= taosArrayGetSize(pReader->aBlockL)) { - pReader->pBlockL = NULL; + if (pReader->iBlockL >= taosArrayGetSize(pReader->aSstBlk)) { + pReader->pSstBlk = NULL; break; } - pReader->pBlockL = (SBlockL*)taosArrayGet(pReader->aBlockL, pReader->iBlockL); - if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { + pReader->pSstBlk = (SSstBlk*)taosArrayGet(pReader->aSstBlk, pReader->iBlockL); + if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) { // TODO break; } @@ -101,8 +101,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { } while (true) { - if (pReader->pBlockIdx && pReader->pBlockL) { - TABLEID id = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; + if (pReader->pBlockIdx && pReader->pSstBlk) { + TABLEID id = {.suid = pReader->pSstBlk->suid, .uid = pReader->pSstBlk->minUid}; ASSERT(0); @@ -142,18 +142,18 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { } if (*ppData) goto _exit; - } else if (pReader->pBlockL) { - while (pReader->pBlockL) { - if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { + } else if (pReader->pSstBlk) { + while (pReader->pSstBlk) { + if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) { // load data (todo) } // next pReader->iBlockL++; - if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) { - pReader->pBlockL = (SBlockL*)taosArrayGetSize(pReader->aBlockL); + if (pReader->iBlockL < taosArrayGetSize(pReader->aSstBlk)) { + pReader->pSstBlk = (SSstBlk*)taosArrayGetSize(pReader->aSstBlk); } else { - pReader->pBlockL = NULL; + pReader->pSstBlk = NULL; } if (*ppData) goto _exit; @@ -298,8 +298,8 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pReader->aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pReader->aBlockL == NULL) { + pReader->aSstBlk = taosArrayInit(0, sizeof(SSstBlk)); + if (pReader->aSstBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -338,7 +338,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { if (pReader->pDataFReader) { tsdbDataFReaderClose(&pReader->pDataFReader); } - taosArrayDestroy(pReader->aBlockL); + taosArrayDestroy(pReader->aSstBlk); taosArrayDestroy(pReader->aBlockIdx); tMapDataClear(&pReader->mBlock); tBlockDataDestroy(&pReader->oBlockData, 1); @@ -431,7 +431,7 @@ struct STsdbSnapWriter { SBlockData* pBlockData; int32_t iRow; SBlockData bDataR; - SArray* aBlockL; // SArray + SArray* aSstBlk; // SArray int32_t iBlockL; SBlockData lDataR; @@ -443,7 +443,7 @@ struct STsdbSnapWriter { SMapData mBlockW; // SMapData SArray* aBlockIdxW; // SArray - SArray* aBlockLW; // SArray + SArray* aBlockLW; // SArray // for del file SDelFReader* pDelFReader; @@ -845,7 +845,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { // write remain stuff if (taosArrayGetSize(pWriter->aBlockLW) > 0) { - code = tsdbWriteBlockL(pWriter->pDataFWriter, pWriter->aBlockIdxW); + code = tsdbWriteSstBlk(pWriter->pDataFWriter, pWriter->aBlockIdxW); if (code) goto _err; } @@ -911,12 +911,12 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx); if (code) goto _err; - code = tsdbReadBlockL(pWriter->pDataFReader, 0, pWriter->aBlockL); + code = tsdbReadSstBlk(pWriter->pDataFReader, 0, pWriter->aSstBlk); if (code) goto _err; } else { ASSERT(pWriter->pDataFReader == NULL); taosArrayClear(pWriter->aBlockIdx); - taosArrayClear(pWriter->aBlockL); + taosArrayClear(pWriter->aSstBlk); } pWriter->iBlockIdx = 0; pWriter->pBlockIdx = NULL; @@ -1147,8 +1147,8 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = tBlockDataCreate(&pWriter->bDataR); if (code) goto _err; - pWriter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pWriter->aBlockL == NULL) { + pWriter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk)); + if (pWriter->aSstBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -1161,7 +1161,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr code = tBlockDataCreate(&pWriter->bDataW); if (code) goto _err; - pWriter->aBlockLW = taosArrayInit(0, sizeof(SBlockL)); + pWriter->aBlockLW = taosArrayInit(0, sizeof(SSstBlk)); if (pWriter->aBlockLW == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index f1e09c7f29..2a3ffd6089 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -214,7 +214,7 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) { int32_t tCmprBlockL(void const *lhs, void const *rhs) { SBlockIdx *lBlockIdx = (SBlockIdx *)lhs; - SBlockL *rBlockL = (SBlockL *)rhs; + SSstBlk *rBlockL = (SSstBlk *)rhs; if (lBlockIdx->suid < rBlockL->suid) { return -1; @@ -311,41 +311,41 @@ bool tBlockHasSma(SBlock *pBlock) { return pBlock->smaInfo.size > 0; } -// SBlockL ====================================================== -int32_t tPutBlockL(uint8_t *p, void *ph) { +// SSstBlk ====================================================== +int32_t tPutSstBlk(uint8_t *p, void *ph) { int32_t n = 0; - SBlockL *pBlockL = (SBlockL *)ph; + SSstBlk *pSstBlk = (SSstBlk *)ph; - n += tPutI64(p ? p + n : p, pBlockL->suid); - n += tPutI64(p ? p + n : p, pBlockL->minUid); - n += tPutI64(p ? p + n : p, pBlockL->maxUid); - n += tPutI64v(p ? p + n : p, pBlockL->minKey); - n += tPutI64v(p ? p + n : p, pBlockL->maxKey); - n += tPutI64v(p ? p + n : p, pBlockL->minVer); - n += tPutI64v(p ? p + n : p, pBlockL->maxVer); - n += tPutI32v(p ? p + n : p, pBlockL->nRow); - n += tPutI64v(p ? p + n : p, pBlockL->bInfo.offset); - n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szBlock); - n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szKey); + n += tPutI64(p ? p + n : p, pSstBlk->suid); + n += tPutI64(p ? p + n : p, pSstBlk->minUid); + n += tPutI64(p ? p + n : p, pSstBlk->maxUid); + n += tPutI64v(p ? p + n : p, pSstBlk->minKey); + n += tPutI64v(p ? p + n : p, pSstBlk->maxKey); + n += tPutI64v(p ? p + n : p, pSstBlk->minVer); + n += tPutI64v(p ? p + n : p, pSstBlk->maxVer); + n += tPutI32v(p ? p + n : p, pSstBlk->nRow); + n += tPutI64v(p ? p + n : p, pSstBlk->bInfo.offset); + n += tPutI32v(p ? p + n : p, pSstBlk->bInfo.szBlock); + n += tPutI32v(p ? p + n : p, pSstBlk->bInfo.szKey); return n; } -int32_t tGetBlockL(uint8_t *p, void *ph) { +int32_t tGetSstBlk(uint8_t *p, void *ph) { int32_t n = 0; - SBlockL *pBlockL = (SBlockL *)ph; + SSstBlk *pSstBlk = (SSstBlk *)ph; - n += tGetI64(p + n, &pBlockL->suid); - n += tGetI64(p + n, &pBlockL->minUid); - n += tGetI64(p + n, &pBlockL->maxUid); - n += tGetI64v(p + n, &pBlockL->minKey); - n += tGetI64v(p + n, &pBlockL->maxKey); - n += tGetI64v(p + n, &pBlockL->minVer); - n += tGetI64v(p + n, &pBlockL->maxVer); - n += tGetI32v(p + n, &pBlockL->nRow); - n += tGetI64v(p + n, &pBlockL->bInfo.offset); - n += tGetI32v(p + n, &pBlockL->bInfo.szBlock); - n += tGetI32v(p + n, &pBlockL->bInfo.szKey); + n += tGetI64(p + n, &pSstBlk->suid); + n += tGetI64(p + n, &pSstBlk->minUid); + n += tGetI64(p + n, &pSstBlk->maxUid); + n += tGetI64v(p + n, &pSstBlk->minKey); + n += tGetI64v(p + n, &pSstBlk->maxKey); + n += tGetI64v(p + n, &pSstBlk->minVer); + n += tGetI64v(p + n, &pSstBlk->maxVer); + n += tGetI32v(p + n, &pSstBlk->nRow); + n += tGetI64v(p + n, &pSstBlk->bInfo.offset); + n += tGetI32v(p + n, &pSstBlk->bInfo.szBlock); + n += tGetI32v(p + n, &pSstBlk->bInfo.szKey); return n; } From aec4297eb46182d4fb59e8a1cda4d1db0000559c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 11:16:23 +0800 Subject: [PATCH 082/102] refact code --- source/dnode/vnode/src/inc/tsdb.h | 18 ++-- source/dnode/vnode/src/tsdb/tsdbCache.c | 6 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 54 +++++----- source/dnode/vnode/src/tsdb/tsdbRead.c | 72 +++++++------- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 32 +++--- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 44 ++++----- source/dnode/vnode/src/tsdb/tsdbUtil.c | 98 +++++++++---------- 7 files changed, 162 insertions(+), 162 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 5a9d6c43af..d6bc3385ed 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -42,7 +42,7 @@ typedef struct SMemTable SMemTable; typedef struct STbDataIter STbDataIter; typedef struct SMapData SMapData; typedef struct SBlockIdx SBlockIdx; -typedef struct SBlock SBlock; +typedef struct SDataBlk SDataBlk; typedef struct SSstBlk SSstBlk; typedef struct SColData SColData; typedef struct SDiskDataHdr SDiskDataHdr; @@ -114,12 +114,12 @@ int32_t tTABLEIDCmprFn(const void *p1, const void *p2); int32_t tPutBlockCol(uint8_t *p, void *ph); int32_t tGetBlockCol(uint8_t *p, void *ph); int32_t tBlockColCmprFn(const void *p1, const void *p2); -// SBlock -void tBlockReset(SBlock *pBlock); -int32_t tPutBlock(uint8_t *p, void *ph); -int32_t tGetBlock(uint8_t *p, void *ph); +// SDataBlk +void tBlockReset(SDataBlk *pBlock); +int32_t tPutDataBlk(uint8_t *p, void *ph); +int32_t tGetDataBlk(uint8_t *p, void *ph); int32_t tBlockCmprFn(const void *p1, const void *p2); -bool tBlockHasSma(SBlock *pBlock); +bool tBlockHasSma(SDataBlk *pBlock); // SSstBlk int32_t tPutSstBlk(uint8_t *p, void *ph); int32_t tGetSstBlk(uint8_t *p, void *ph); @@ -265,8 +265,8 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk); -int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData); +int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg); +int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData); int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData); // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); @@ -427,7 +427,7 @@ struct SSmaInfo { int32_t size; }; -struct SBlock { +struct SDataBlk { TSDBKEY minKey; TSDBKEY maxKey; int64_t minVer; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index a0d52b8b3f..b12fb15798 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -527,7 +527,7 @@ typedef struct SFSNextRowIter { SMapData blockMap; int32_t nBlock; int32_t iBlock; - SBlock block; + SDataBlk block; SBlockData blockData; SBlockData *pBlockData; int32_t nRow; @@ -602,13 +602,13 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } case SFSNEXTROW_BLOCKDATA: if (state->iBlock >= 0) { - SBlock block = {0}; + SDataBlk block = {0}; tBlockReset(&block); // tBlockDataReset(&state->blockData); tBlockDataReset(state->pBlockData); - tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetBlock); + tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ tBlockDataReset(state->pBlockData); code = tBlockDataInit(state->pBlockData, state->suid, state->uid, state->pTSchema); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index caab533f6e..388e9ce4f1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -64,7 +64,7 @@ typedef struct { SArray *aBlockIdx; // SArray int32_t iBlockIdx; SBlockIdx *pBlockIdx; - SMapData mBlock; // SMapData + SMapData mBlock; // SMapData SBlockData bData; } dReader; struct { @@ -78,7 +78,7 @@ typedef struct { SDataFWriter *pWriter; SArray *aBlockIdx; // SArray SArray *aSstBlk; // SArray - SMapData mBlock; // SMapData + SMapData mBlock; // SMapData SBlockData bData; SBlockData bDatal; } dWriter; @@ -562,7 +562,7 @@ _err: static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) { int32_t code = 0; SBlockData *pBlockData = &pCommitter->dWriter.bData; - SBlock block; + SDataBlk block; ASSERT(pBlockData->nRow > 0); @@ -597,8 +597,8 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) { ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0); if (code) goto _err; - // put SBlock - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); + // put SDataBlk + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutDataBlk); if (code) goto _err; // clear @@ -1098,7 +1098,7 @@ _exit: return code; } -static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SBlock *pBlock) { +static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { int32_t code = 0; SBlockData *pBlockData = &pCommitter->dWriter.bData; SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); @@ -1122,7 +1122,7 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SBlock *pBlock) { pRowInfo = NULL; } else { TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); - if (tsdbKeyCmprFn(&tKey, &pBlock->minKey) >= 0) pRowInfo = NULL; + if (tsdbKeyCmprFn(&tKey, &pDataBlk->minKey) >= 0) pRowInfo = NULL; } } @@ -1144,14 +1144,14 @@ _err: return code; } -static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SBlock *pBlock) { +static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { int32_t code = 0; SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; SBlockData *pBDataR = &pCommitter->dReader.bData; SBlockData *pBDataW = &pCommitter->dWriter.bData; - code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBDataR); + code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR); if (code) goto _err; tBlockDataClear(pBDataW); @@ -1188,7 +1188,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SBlock *pBlock) { pRowInfo = NULL; } else { TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); - if (tsdbKeyCmprFn(&tKey, &pBlock->maxKey) > 0) pRowInfo = NULL; + if (tsdbKeyCmprFn(&tKey, &pDataBlk->maxKey) > 0) pRowInfo = NULL; } } } else { @@ -1237,57 +1237,57 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0); if (pBlockIdx && pBlockIdx->suid == id.suid && pBlockIdx->uid == id.uid) { int32_t iBlock = 0; - SBlock block; - SBlock *pBlock = █ + SDataBlk block; + SDataBlk *pDataBlk = █ SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); ASSERT(pRowInfo->suid == id.suid && pRowInfo->uid == id.uid); - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - while (pBlock && pRowInfo) { - SBlock tBlock = {.minKey = TSDBROW_KEY(&pRowInfo->row), .maxKey = TSDBROW_KEY(&pRowInfo->row)}; - int32_t c = tBlockCmprFn(pBlock, &tBlock); + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); + while (pDataBlk && pRowInfo) { + SDataBlk tBlock = {.minKey = TSDBROW_KEY(&pRowInfo->row), .maxKey = TSDBROW_KEY(&pRowInfo->row)}; + int32_t c = tBlockCmprFn(pDataBlk, &tBlock); if (c < 0) { - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); if (code) goto _err; iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); } else { - pBlock = NULL; + pDataBlk = NULL; } } else if (c > 0) { - code = tsdbCommitAheadBlock(pCommitter, pBlock); + code = tsdbCommitAheadBlock(pCommitter, pDataBlk); if (code) goto _err; pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; } else { - code = tsdbCommitMergeBlock(pCommitter, pBlock); + code = tsdbCommitMergeBlock(pCommitter, pDataBlk); if (code) goto _err; iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); } else { - pBlock = NULL; + pDataBlk = NULL; } pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; } } - while (pBlock) { - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); + while (pDataBlk) { + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); if (code) goto _err; iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); } else { - pBlock = NULL; + pDataBlk = NULL; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ef0c23fd58..828bc8469a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -109,7 +109,7 @@ typedef struct SDataBlockIter { int32_t index; SArray* blockList; // SArray int32_t order; - SBlock block; // current SBlock data + SDataBlk block; // current SDataBlk data SHashObj* pTableMap; } SDataBlockIter; @@ -590,8 +590,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN sizeInDisk += pScanInfo->mapData.nData; for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { - SBlock block = {0}; - tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetBlock); + SDataBlk block = {0}; + tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetDataBlk); // 1. time range check if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { @@ -665,7 +665,7 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { return pBlockInfo; } -static SBlock* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } +static SDataBlk* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { SReaderStatus* pStatus = &pReader->status; @@ -673,7 +673,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn SBlockData* pBlockData = &pStatus->fileBlockData; SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - SBlock* pBlock = getCurrentBlock(pBlockIter); + SDataBlk* pBlock = getCurrentBlock(pBlockIter); SSDataBlock* pResBlock = pReader->pResBlock; int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock); @@ -758,8 +758,8 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; ASSERT(pBlockInfo != NULL); - SBlock* pBlock = getCurrentBlock(pBlockIter); - int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); + SDataBlk* pBlock = getCurrentBlock(pBlockIter); + int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, code:%s %s", @@ -836,7 +836,7 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) { if (pBlockInfo != NULL) { STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); - tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetBlock); + tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetDataBlk); } #if 0 @@ -887,12 +887,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; - SBlock block = {0}; + SDataBlk block = {0}; for (int32_t k = 0; k < num; ++k) { SBlockOrderWrapper wrapper = {0}; int32_t* mapDataIndex = taosArrayGet(pTableScanInfo->pBlockList, k); - tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetBlock); + tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetDataBlk); wrapper.uid = pTableScanInfo->uid; wrapper.offset = block.aSubBlock[0].offset; @@ -981,15 +981,15 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) { /** * This is an two rectangles overlap cases. */ -static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SBlock* pBlock) { +static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SDataBlk* pBlock) { return (pWindow->ekey < pBlock->maxKey.ts && pWindow->ekey >= pBlock->minKey.ts) || (pWindow->skey > pBlock->minKey.ts && pWindow->skey <= pBlock->maxKey.ts) || (pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) || (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); } -static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, - int32_t* nextIndex, int32_t order) { +static SDataBlk* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, + int32_t* nextIndex, int32_t order) { bool asc = ASCENDING_TRAVERSE(order); if (asc && pFBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) { return NULL; @@ -1002,10 +1002,10 @@ static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STab int32_t step = asc ? 1 : -1; *nextIndex = pFBlockInfo->tbBlockIdx + step; - SBlock* pBlock = taosMemoryCalloc(1, sizeof(SBlock)); - int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); + SDataBlk* pBlock = taosMemoryCalloc(1, sizeof(SDataBlk)); + int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); - tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetBlock); + tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetDataBlk); return pBlock; } @@ -1048,7 +1048,7 @@ static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t return TSDB_CODE_SUCCESS; } -static bool overlapWithNeighborBlock(SBlock* pBlock, SBlock* pNeighbor, int32_t order) { +static bool overlapWithNeighborBlock(SDataBlk* pBlock, SDataBlk* pNeighbor, int32_t order) { // it is the last block in current file, no chance to overlap with neighbor blocks. if (ASCENDING_TRAVERSE(order)) { return pBlock->maxKey.ts == pNeighbor->minKey.ts; @@ -1057,19 +1057,19 @@ static bool overlapWithNeighborBlock(SBlock* pBlock, SBlock* pNeighbor, int32_t } } -static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SBlock* pBlock) { +static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SDataBlk* pBlock) { bool ascScan = ASCENDING_TRAVERSE(order); return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->minKey.ts)) || (!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->maxKey.ts)); } -static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVerRange) { +static bool keyOverlapFileBlock(TSDBKEY key, SDataBlk* pBlock, SVersionRange* pVerRange) { return (key.ts >= pBlock->minKey.ts && key.ts <= pBlock->maxKey.ts) && (pBlock->maxVer >= pVerRange->minVer) && (pBlock->minVer <= pVerRange->maxVer); } -static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) { +static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock) { size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) { @@ -1103,7 +1103,7 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons return false; } -static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) { +static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock, int32_t order) { if (pBlockScanInfo->delSkyline == NULL) { return false; } @@ -1138,10 +1138,10 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBl // 3. current timestamp should not be overlap with each other // 4. output buffer should be large enough to hold all rows in current block // 5. delete info should not overlap with current block data -static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SBlock* pBlock, +static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SDataBlk* pBlock, STableBlockScanInfo* pScanInfo, TSDBKEY key, SLastBlockReader* pLastBlockReader) { - int32_t neighborIndex = 0; - SBlock* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order); + int32_t neighborIndex = 0; + SDataBlk* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order); // overlap with neighbor bool overlapWithNeighbor = false; @@ -1948,7 +1948,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pDumpInfo->rowIndex += step; - SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); break; @@ -1967,7 +1967,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { - SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); break; } @@ -2320,8 +2320,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { } static int32_t doBuildDataBlock(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; - SBlock* pBlock = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SDataBlk* pBlock = NULL; SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -2418,7 +2418,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { // set the correct start position in case of the first/last file block, according to the query time window static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - SBlock* pBlock = getCurrentBlock(pBlockIter); + SDataBlk* pBlock = getCurrentBlock(pBlockIter); SReaderStatus* pStatus = &pReader->status; @@ -2789,7 +2789,7 @@ typedef enum { CHECK_FILEBLOCK_QUIT = 0x2, } CHECK_FILEBLOCK_STATE; -static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, SBlock* pBlock, +static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, SDataBlk* pBlock, SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, int64_t key, CHECK_FILEBLOCK_STATE* state) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2798,8 +2798,8 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn *state = CHECK_FILEBLOCK_QUIT; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - int32_t nextIndex = -1; - SBlock* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order); + int32_t nextIndex = -1; + SDataBlk* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order); if (pNeighborBlock == NULL) { // do nothing return 0; } @@ -2863,7 +2863,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc CHECK_FILEBLOCK_STATE st; SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); - SBlock* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter); + SDataBlk* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter); checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st); if (st == CHECK_FILEBLOCK_QUIT) { break; @@ -3447,8 +3447,8 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); - SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); - int64_t stime = taosGetTimestampUs(); + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + int64_t stime = taosGetTimestampUs(); SBlockLoadSuppInfo* pSup = &pReader->suppInfo; @@ -3629,7 +3629,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa while (true) { if (hasNext) { - SBlock* pBlock = getCurrentBlock(pBlockIter); + SDataBlk* pBlock = getCurrentBlock(pBlockIter); int32_t numOfRows = pBlock->nRow; pTableBlockInfo->totalRows += numOfRows; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index d9c85f55e0..dbaf9b234c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -677,9 +677,9 @@ _err: return code; } -int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg) { +int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aColumnDataAgg) { int32_t code = 0; - SSmaInfo *pSmaInfo = &pBlock->smaInfo; + SSmaInfo *pSmaInfo = &pDataBlk->smaInfo; ASSERT(pSmaInfo->size > 0); @@ -843,13 +843,13 @@ _err: return code; } -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData) { +int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) { int32_t code = 0; - code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, pBlockData); + code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], 0, pBlockData); if (code) goto _err; - if (pBlock->nSubBlock > 1) { + if (pDataBlk->nSubBlock > 1) { SBlockData bData1; SBlockData bData2; @@ -863,8 +863,8 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl tBlockDataInitEx(&bData1, pBlockData); tBlockDataInitEx(&bData2, pBlockData); - for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[iSubBlock], 0, &bData1); + for (int32_t iSubBlock = 1; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { + code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], 0, &bData1); if (code) { tBlockDataDestroy(&bData1, 1); tBlockDataDestroy(&bData2, 1); @@ -1355,28 +1355,28 @@ _err: return code; } -static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { +static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SDataBlk *pDataBlk) { for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; if (iRow == 0) { - if (tsdbKeyCmprFn(&pBlock->minKey, &key) > 0) { - pBlock->minKey = key; + if (tsdbKeyCmprFn(&pDataBlk->minKey, &key) > 0) { + pDataBlk->minKey = key; } } else { if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) { - pBlock->hasDup = 1; + pDataBlk->hasDup = 1; } } - if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&pBlock->maxKey, &key) < 0) { - pBlock->maxKey = key; + if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&pDataBlk->maxKey, &key) < 0) { + pDataBlk->maxKey = key; } - pBlock->minVer = TMIN(pBlock->minVer, key.version); - pBlock->maxVer = TMAX(pBlock->maxVer, key.version); + pDataBlk->minVer = TMIN(pDataBlk->minVer, key.version); + pDataBlk->maxVer = TMAX(pDataBlk->maxVer, key.version); } - pBlock->nRow += pBlockData->nRow; + pDataBlk->nRow += pBlockData->nRow; } static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 62c279ef06..02f2205875 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -33,7 +33,7 @@ struct STsdbSnapReader { int32_t iBlockIdx; int32_t iBlockL; - SMapData mBlock; // SMapData + SMapData mBlock; // SMapData int32_t iBlock; SBlockData oBlockData; SBlockData nBlockData; @@ -115,8 +115,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { // } } else if (pReader->pBlockIdx) { while (pReader->iBlock < pReader->mBlock.nItem) { - SBlock block; - tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetBlock); + SDataBlk block; + tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetDataBlk); if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) { // load data (todo) @@ -426,7 +426,7 @@ struct STsdbSnapWriter { SArray* aBlockIdx; // SArray int32_t iBlockIdx; SBlockIdx* pBlockIdx; - SMapData mBlock; // SMapData + SMapData mBlock; // SMapData int32_t iBlock; SBlockData* pBlockData; int32_t iRow; @@ -437,11 +437,11 @@ struct STsdbSnapWriter { SDataFWriter* pDataFWriter; SBlockIdx* pBlockIdxW; // NULL when no committing table - SBlock blockW; + SDataBlk blockW; SBlockData bDataW; SBlockIdx blockIdxW; - SMapData mBlockW; // SMapData + SMapData mBlockW; // SMapData SArray* aBlockIdxW; // SArray SArray* aBlockLW; // SArray @@ -475,7 +475,7 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { // &pWriter->blockW, pWriter->cmprAlg); if (code) goto _err; - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk); if (code) goto _err; tBlockReset(&pWriter->blockW); @@ -499,15 +499,15 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { // &pWriter->blockW, pWriter->cmprAlg); // if (code) goto _err; - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk); if (code) goto _err; } while (true) { if (pWriter->iBlock >= pWriter->mBlock.nItem) break; - SBlock block; - tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); + SDataBlk block; + tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk); // if (block.last) { // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); @@ -520,13 +520,13 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { // if (code) goto _err; // } - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk); if (code) goto _err; pWriter->iBlock++; } - // SBlock + // SDataBlk // code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); // if (code) goto _err; @@ -553,10 +553,10 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p if (code) goto _err; // SBlockData - SBlock block; + SDataBlk block; tMapDataReset(&pWriter->mBlockW); for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) { - tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock); + tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetDataBlk); // if (block.last) { // code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); @@ -570,11 +570,11 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p // if (code) goto _err; // } - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk); if (code) goto _err; } - // SBlock + // SDataBlk SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx); if (code) goto _err; @@ -642,10 +642,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { while (true) { if (pWriter->iBlock >= pWriter->mBlock.nItem) break; - SBlock block; - int32_t c; + SDataBlk block; + int32_t c; - tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); + tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk); // if (block.last) { // pWriter->pBlockData = &pWriter->bDataR; @@ -668,14 +668,14 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { // &pWriter->blockW, pWriter->cmprAlg); // if (code) goto _err; - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk); if (code) goto _err; tBlockReset(&pWriter->blockW); tBlockDataClear(&pWriter->bDataW); } - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk); if (code) goto _err; pWriter->iBlock++; @@ -719,7 +719,7 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { // &pWriter->blockW, pWriter->cmprAlg); // if (code) goto _err; - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk); if (code) goto _err; tBlockReset(&pWriter->blockW); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 2a3ffd6089..62dfb5157f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -231,69 +231,69 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs) { return 0; } -// SBlock ====================================================== -void tBlockReset(SBlock *pBlock) { - *pBlock = (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; +// SDataBlk ====================================================== +void tBlockReset(SDataBlk *pDataBlk) { + *pDataBlk = (SDataBlk){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; } -int32_t tPutBlock(uint8_t *p, void *ph) { - int32_t n = 0; - SBlock *pBlock = (SBlock *)ph; +int32_t tPutDataBlk(uint8_t *p, void *ph) { + int32_t n = 0; + SDataBlk *pDataBlk = (SDataBlk *)ph; - n += tPutI64v(p ? p + n : p, pBlock->minKey.version); - n += tPutI64v(p ? p + n : p, pBlock->minKey.ts); - n += tPutI64v(p ? p + n : p, pBlock->maxKey.version); - n += tPutI64v(p ? p + n : p, pBlock->maxKey.ts); - n += tPutI64v(p ? p + n : p, pBlock->minVer); - n += tPutI64v(p ? p + n : p, pBlock->maxVer); - n += tPutI32v(p ? p + n : p, pBlock->nRow); - n += tPutI8(p ? p + n : p, pBlock->hasDup); - n += tPutI8(p ? p + n : p, pBlock->nSubBlock); - for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - n += tPutI64v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].offset); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szBlock); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szKey); + n += tPutI64v(p ? p + n : p, pDataBlk->minKey.version); + n += tPutI64v(p ? p + n : p, pDataBlk->minKey.ts); + n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.version); + n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.ts); + n += tPutI64v(p ? p + n : p, pDataBlk->minVer); + n += tPutI64v(p ? p + n : p, pDataBlk->maxVer); + n += tPutI32v(p ? p + n : p, pDataBlk->nRow); + n += tPutI8(p ? p + n : p, pDataBlk->hasDup); + n += tPutI8(p ? p + n : p, pDataBlk->nSubBlock); + for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { + n += tPutI64v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].offset); + n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szBlock); + n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szKey); } - if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { - n += tPutI64v(p ? p + n : p, pBlock->smaInfo.offset); - n += tPutI32v(p ? p + n : p, pBlock->smaInfo.size); + if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) { + n += tPutI64v(p ? p + n : p, pDataBlk->smaInfo.offset); + n += tPutI32v(p ? p + n : p, pDataBlk->smaInfo.size); } return n; } -int32_t tGetBlock(uint8_t *p, void *ph) { - int32_t n = 0; - SBlock *pBlock = (SBlock *)ph; +int32_t tGetDataBlk(uint8_t *p, void *ph) { + int32_t n = 0; + SDataBlk *pDataBlk = (SDataBlk *)ph; - n += tGetI64v(p + n, &pBlock->minKey.version); - n += tGetI64v(p + n, &pBlock->minKey.ts); - n += tGetI64v(p + n, &pBlock->maxKey.version); - n += tGetI64v(p + n, &pBlock->maxKey.ts); - n += tGetI64v(p + n, &pBlock->minVer); - n += tGetI64v(p + n, &pBlock->maxVer); - n += tGetI32v(p + n, &pBlock->nRow); - n += tGetI8(p + n, &pBlock->hasDup); - n += tGetI8(p + n, &pBlock->nSubBlock); - for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - n += tGetI64v(p + n, &pBlock->aSubBlock[iSubBlock].offset); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szBlock); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szKey); + n += tGetI64v(p + n, &pDataBlk->minKey.version); + n += tGetI64v(p + n, &pDataBlk->minKey.ts); + n += tGetI64v(p + n, &pDataBlk->maxKey.version); + n += tGetI64v(p + n, &pDataBlk->maxKey.ts); + n += tGetI64v(p + n, &pDataBlk->minVer); + n += tGetI64v(p + n, &pDataBlk->maxVer); + n += tGetI32v(p + n, &pDataBlk->nRow); + n += tGetI8(p + n, &pDataBlk->hasDup); + n += tGetI8(p + n, &pDataBlk->nSubBlock); + for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { + n += tGetI64v(p + n, &pDataBlk->aSubBlock[iSubBlock].offset); + n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szBlock); + n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szKey); } - if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { - n += tGetI64v(p + n, &pBlock->smaInfo.offset); - n += tGetI32v(p + n, &pBlock->smaInfo.size); + if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) { + n += tGetI64v(p + n, &pDataBlk->smaInfo.offset); + n += tGetI32v(p + n, &pDataBlk->smaInfo.size); } else { - pBlock->smaInfo.offset = 0; - pBlock->smaInfo.size = 0; + pDataBlk->smaInfo.offset = 0; + pDataBlk->smaInfo.size = 0; } return n; } int32_t tBlockCmprFn(const void *p1, const void *p2) { - SBlock *pBlock1 = (SBlock *)p1; - SBlock *pBlock2 = (SBlock *)p2; + SDataBlk *pBlock1 = (SDataBlk *)p1; + SDataBlk *pBlock2 = (SDataBlk *)p2; if (tsdbKeyCmprFn(&pBlock1->maxKey, &pBlock2->minKey) < 0) { return -1; @@ -304,11 +304,11 @@ int32_t tBlockCmprFn(const void *p1, const void *p2) { return 0; } -bool tBlockHasSma(SBlock *pBlock) { - if (pBlock->nSubBlock > 1) return false; - if (pBlock->hasDup) return false; +bool tBlockHasSma(SDataBlk *pDataBlk) { + if (pDataBlk->nSubBlock > 1) return false; + if (pDataBlk->hasDup) return false; - return pBlock->smaInfo.size > 0; + return pDataBlk->smaInfo.size > 0; } // SSstBlk ====================================================== From 0eb29455aeb5d590edcf1199027ecf3c142e67fa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 11:19:34 +0800 Subject: [PATCH 083/102] refact code --- source/dnode/vnode/src/inc/tsdb.h | 6 +++--- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 8 ++++---- source/dnode/vnode/src/tsdb/tsdbUtil.c | 6 +++--- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d6bc3385ed..92c6ecd07c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -115,11 +115,11 @@ int32_t tPutBlockCol(uint8_t *p, void *ph); int32_t tGetBlockCol(uint8_t *p, void *ph); int32_t tBlockColCmprFn(const void *p1, const void *p2); // SDataBlk -void tBlockReset(SDataBlk *pBlock); +void tDataBlkReset(SDataBlk *pBlock); int32_t tPutDataBlk(uint8_t *p, void *ph); int32_t tGetDataBlk(uint8_t *p, void *ph); -int32_t tBlockCmprFn(const void *p1, const void *p2); -bool tBlockHasSma(SDataBlk *pBlock); +int32_t tDataBlkCmprFn(const void *p1, const void *p2); +bool tDataBlkHasSma(SDataBlk *pDataBlk); // SSstBlk int32_t tPutSstBlk(uint8_t *p, void *ph); int32_t tGetSstBlk(uint8_t *p, void *ph); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index b12fb15798..64caff1542 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -604,7 +604,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (state->iBlock >= 0) { SDataBlk block = {0}; - tBlockReset(&block); + tDataBlkReset(&block); // tBlockDataReset(&state->blockData); tBlockDataReset(state->pBlockData); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 388e9ce4f1..42ab6ce3b2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -566,7 +566,7 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) { ASSERT(pBlockData->nRow > 0); - tBlockReset(&block); + tDataBlkReset(&block); // info block.nRow += pBlockData->nRow; @@ -1246,7 +1246,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); while (pDataBlk && pRowInfo) { SDataBlk tBlock = {.minKey = TSDBROW_KEY(&pRowInfo->row), .maxKey = TSDBROW_KEY(&pRowInfo->row)}; - int32_t c = tBlockCmprFn(pDataBlk, &tBlock); + int32_t c = tDataBlkCmprFn(pDataBlk, &tBlock); if (c < 0) { code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 828bc8469a..358995b8d1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3452,7 +3452,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS SBlockLoadSuppInfo* pSup = &pReader->suppInfo; - if (tBlockHasSma(pBlock)) { + if (tDataBlkHasSma(pBlock)) { code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg); if (code != TSDB_CODE_SUCCESS) { tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 02f2205875..8d19a2ffb8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -478,7 +478,7 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk); if (code) goto _err; - tBlockReset(&pWriter->blockW); + tDataBlkReset(&pWriter->blockW); tBlockDataClear(&pWriter->bDataW); } @@ -671,7 +671,7 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk); if (code) goto _err; - tBlockReset(&pWriter->blockW); + tDataBlkReset(&pWriter->blockW); tBlockDataClear(&pWriter->bDataW); } @@ -722,7 +722,7 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk); if (code) goto _err; - tBlockReset(&pWriter->blockW); + tDataBlkReset(&pWriter->blockW); tBlockDataClear(&pWriter->bDataW); } @@ -803,7 +803,7 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { pWriter->pBlockIdxW->suid = id.suid; pWriter->pBlockIdxW->uid = id.uid; - tBlockReset(&pWriter->blockW); + tDataBlkReset(&pWriter->blockW); tBlockDataReset(&pWriter->bDataW); tMapDataReset(&pWriter->mBlockW); } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 62dfb5157f..6937a27fe4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -232,7 +232,7 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs) { } // SDataBlk ====================================================== -void tBlockReset(SDataBlk *pDataBlk) { +void tDataBlkReset(SDataBlk *pDataBlk) { *pDataBlk = (SDataBlk){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; } @@ -291,7 +291,7 @@ int32_t tGetDataBlk(uint8_t *p, void *ph) { return n; } -int32_t tBlockCmprFn(const void *p1, const void *p2) { +int32_t tDataBlkCmprFn(const void *p1, const void *p2) { SDataBlk *pBlock1 = (SDataBlk *)p1; SDataBlk *pBlock2 = (SDataBlk *)p2; @@ -304,7 +304,7 @@ int32_t tBlockCmprFn(const void *p1, const void *p2) { return 0; } -bool tBlockHasSma(SDataBlk *pDataBlk) { +bool tDataBlkHasSma(SDataBlk *pDataBlk) { if (pDataBlk->nSubBlock > 1) return false; if (pDataBlk->hasDup) return false; From 720f3b2482e552ba1d1554e7f53d823e5beef8b9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Sep 2022 11:20:12 +0800 Subject: [PATCH 084/102] refactor: do some internal refactor. --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 22 ++- source/dnode/vnode/src/tsdb/tsdbRead.c | 184 +++++++++++--------- 3 files changed, 122 insertions(+), 86 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 111c6b5962..918e3e648a 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -639,7 +639,7 @@ typedef struct SMergeTree { struct SLDataIter *pIter; } SMergeTree; -void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange); +int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange); void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter); bool tMergeTreeNext(SMergeTree* pMTree); TSDBROW tMergeTreeGetRow(SMergeTree* pMTree); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 47c09b51e9..256898a597 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -36,6 +36,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t STimeWindow *pTimeWindow, SVersionRange *pRange) { int32_t code = 0; *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); + if (*pIter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } (*pIter)->uid = uid; (*pIter)->timeWindow = *pTimeWindow; @@ -271,16 +275,24 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { } } -void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange) { +int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange) { pMTree->backward = backward; pMTree->pIter = NULL; pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); + if (pMTree->pIterList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); + int32_t code = TSDB_CODE_OUT_OF_MEMORY; struct SLDataIter* pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; for(int32_t i = 0; i < pFReader->pSet->nLastF; ++i) { // open all last file - /*int32_t code = */tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); + code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + bool hasVal = tLDataIterNextRow(pIterList[i]); if (hasVal) { taosArrayPush(pMTree->pIterList, &pIterList[i]); @@ -289,6 +301,12 @@ void tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, tLDataIterClose(pIterList[i]); } } + + return code; + + _end: + tMergeTreeClose(pMTree); + return code; } void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2d70cc2e4d..e77123c880 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1748,6 +1748,70 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* } #endif +static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + if (pBlockScanInfo->iterInit) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = TSDB_CODE_SUCCESS; + + TSDBKEY startKey = {0}; + if (ASCENDING_TRAVERSE(pReader->order)) { + startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer}; + } else { + startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer}; + } + + int32_t backward = (!ASCENDING_TRAVERSE(pReader->order)); + + STbData* d = NULL; + if (pReader->pReadSnap->pMem != NULL) { + d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); + if (d != NULL) { + code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); + if (code == TSDB_CODE_SUCCESS) { + pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL); + + tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + "-%" PRId64 " %s", + pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, d->minKey, d->maxKey, pReader->idStr); + } else { + tsdbError("%p uid:%" PRId64 ", failed to create iterator for imem, code:%s, %s", pReader, pBlockScanInfo->uid, + tstrerror(code), pReader->idStr); + return code; + } + } + } else { + tsdbDebug("%p uid:%" PRId64 ", no data in mem, %s", pReader, pBlockScanInfo->uid, pReader->idStr); + } + + STbData* di = NULL; + if (pReader->pReadSnap->pIMem != NULL) { + di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid); + if (di != NULL) { + code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter); + if (code == TSDB_CODE_SUCCESS) { + pBlockScanInfo->iiter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iiter.iter) != NULL); + + tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + "-%" PRId64 " %s", + pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, di->minKey, di->maxKey, pReader->idStr); + } else { + tsdbError("%p uid:%" PRId64 ", failed to create iterator for mem, code:%s, %s", pReader, pBlockScanInfo->uid, + tstrerror(code), pReader->idStr); + return code; + } + } + } else { + tsdbDebug("%p uid:%" PRId64 ", no data in imem, %s", pReader, pBlockScanInfo->uid, pReader->idStr); + } + + initDelSkylineIterator(pBlockScanInfo, pReader, d, di); + + pBlockScanInfo->iterInit = true; + return TSDB_CODE_SUCCESS; +} + static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { // it is an multi-table data block @@ -1779,25 +1843,20 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, SDataFReader* pFReader) { - // the last block reader has been initialized for this table. - if (pLastBlockReader->uid == uid) { - return true; +static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { + while(1) { + bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); + if (!hasVal) { + return false; + } + + TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBKEY k = TSDBROW_KEY(&row); + if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) { + return true; + } } - if (pLastBlockReader->uid != 0) { - tMergeTreeClose(&pLastBlockReader->mergeTree); - } - - pLastBlockReader->uid = uid; - /*int32_t code = */ tMergeTreeOpen(&pLastBlockReader->mergeTree, (pLastBlockReader->order == TSDB_ORDER_DESC), - pFReader, uid, &pLastBlockReader->window, &pLastBlockReader->verRange); - return tMergeTreeNext(&pLastBlockReader->mergeTree); -} - -static bool nextRowInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { - return tMergeTreeNext(&pLastBlockReader->mergeTree); - #if 0 *(pLastBlockReader->rowIndex) += step; @@ -1854,6 +1913,29 @@ static bool nextRowInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc #endif } +static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader) { + // the last block reader has been initialized for this table. + if (pLastBlockReader->uid == pBlockScanInfo->uid) { + return true; + } + + if (pLastBlockReader->uid != 0) { + tMergeTreeClose(&pLastBlockReader->mergeTree); + } + + initMemDataIterator(pBlockScanInfo, pReader); + pLastBlockReader->uid = pBlockScanInfo->uid; + int32_t code = + tMergeTreeOpen(&pLastBlockReader->mergeTree, (pLastBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, + pBlockScanInfo->uid, &pLastBlockReader->window, &pLastBlockReader->verRange); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + return nextRowFromLastBlocks(pLastBlockReader, pBlockScanInfo); +} + static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); TSDBKEY key = TSDBROW_KEY(&row); @@ -1998,70 +2080,6 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; } -static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { - if (pBlockScanInfo->iterInit) { - return TSDB_CODE_SUCCESS; - } - - int32_t code = TSDB_CODE_SUCCESS; - - TSDBKEY startKey = {0}; - if (ASCENDING_TRAVERSE(pReader->order)) { - startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer}; - } else { - startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer}; - } - - int32_t backward = (!ASCENDING_TRAVERSE(pReader->order)); - - STbData* d = NULL; - if (pReader->pReadSnap->pMem != NULL) { - d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); - if (d != NULL) { - code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); - if (code == TSDB_CODE_SUCCESS) { - pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL); - - tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 " %s", - pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, d->minKey, d->maxKey, pReader->idStr); - } else { - tsdbError("%p uid:%" PRId64 ", failed to create iterator for imem, code:%s, %s", pReader, pBlockScanInfo->uid, - tstrerror(code), pReader->idStr); - return code; - } - } - } else { - tsdbDebug("%p uid:%" PRId64 ", no data in mem, %s", pReader, pBlockScanInfo->uid, pReader->idStr); - } - - STbData* di = NULL; - if (pReader->pReadSnap->pIMem != NULL) { - di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid); - if (di != NULL) { - code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter); - if (code == TSDB_CODE_SUCCESS) { - pBlockScanInfo->iiter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iiter.iter) != NULL); - - tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 " %s", - pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, di->minKey, di->maxKey, pReader->idStr); - } else { - tsdbError("%p uid:%" PRId64 ", failed to create iterator for mem, code:%s, %s", pReader, pBlockScanInfo->uid, - tstrerror(code), pReader->idStr); - return code; - } - } - } else { - tsdbDebug("%p uid:%" PRId64 ", no data in imem, %s", pReader, pBlockScanInfo->uid, pReader->idStr); - } - - initDelSkylineIterator(pBlockScanInfo, pReader, d, di); - - pBlockScanInfo->iterInit = true; - return TSDB_CODE_SUCCESS; -} - int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData, STbData* piMemTbData) { if (pBlockScanInfo->delSkyline != NULL) { @@ -2296,7 +2314,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { while (1) { // load the last data block of current table STableBlockScanInfo* pScanInfo = pStatus->pTableIter; - bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo->uid, pReader->pFileReader); + bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); if (!hasVal) { bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); if (!hasNexTable) { @@ -2879,7 +2897,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger) { - while (nextRowInLastBlock(pLastBlockReader, pScanInfo)) { + while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); From a170796f663e1dc217c11e49f3051779e3a4aae3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Sep 2022 14:34:30 +0800 Subject: [PATCH 085/102] fix(query): fix invalid read/write. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 13 ++++++------- source/libs/executor/inc/executil.h | 4 +--- source/libs/executor/src/timewindowoperator.c | 11 +++++++++-- tests/script/tsim/parser/slimit_alter_tags.sim | 1 + 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d76787915a..7f492f73b0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2244,9 +2244,12 @@ static void extractOrderedTableUidList(SUidOrderCheckInfo* pOrderCheckInfo, SRea } static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderStatus* pStatus) { - if (pOrderCheckInfo->tableUidList == NULL) { - int32_t total = taosHashGetSize(pStatus->pTableMap); + int32_t total = taosHashGetSize(pStatus->pTableMap); + if (total == 0) { + return TSDB_CODE_SUCCESS; + } + if (pOrderCheckInfo->tableUidList == NULL) { pOrderCheckInfo->currentIndex = 0; pOrderCheckInfo->tableUidList = taosMemoryMalloc(total * sizeof(uint64_t)); if (pOrderCheckInfo->tableUidList == NULL) { @@ -2254,21 +2257,17 @@ static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderSt } extractOrderedTableUidList(pOrderCheckInfo, pStatus); - uint64_t uid = pOrderCheckInfo->tableUidList[0]; pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); } else { if (pStatus->pTableIter == NULL) { // it is the last block of a new file - // ASSERT(pOrderCheckInfo->currentIndex == taosHashGetSize(pStatus->pTableMap)); - pOrderCheckInfo->currentIndex = 0; uint64_t uid = pOrderCheckInfo->tableUidList[pOrderCheckInfo->currentIndex]; pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); // the tableMap has already updated if (pStatus->pTableIter == NULL) { - void* p = - taosMemoryRealloc(pOrderCheckInfo->tableUidList, taosHashGetSize(pStatus->pTableMap) * sizeof(uint64_t)); + void* p = taosMemoryRealloc(pOrderCheckInfo->tableUidList, total * sizeof(uint64_t)); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 9e7fcc2227..0722c2b306 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -87,9 +87,7 @@ struct SqlFunctionCtx; size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); void initResultRowInfo(SResultRowInfo* pResultRowInfo); - -void initResultRow(SResultRow* pResultRow); -void closeResultRow(SResultRow* pResultRow); +void closeResultRow(SResultRow* pResultRow); struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 26667cd64f..0304dbdbc8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -609,6 +609,7 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num while (1) { SListNode* pn = tdListGetHead(pResultRowInfo->openWindow); SOpenWindowInfo* pOpenWin = (SOpenWindowInfo *)pn->data; + uint64_t groupId = pOpenWin->groupId; SResultRowPosition* p1 = &pOpenWin->pos; if (p->pageId == p1->pageId && p->offset == p1->offset) { @@ -621,7 +622,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num if (pr->closed) { ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)); - tdListPopHead(pResultRowInfo->openWindow); + SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow); + taosMemoryFree(pNode); continue; } @@ -650,7 +652,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { closeResultRow(pr); - tdListPopHead(pResultRowInfo->openWindow); + SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow); + taosMemoryFree(pNode); } else { // the remains are can not be closed yet. break; } @@ -1730,6 +1733,10 @@ void destroyIntervalOperatorInfo(void* param) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); + cleanupExprSupp(&pInfo->scalarSupp); + + tdListFree(pInfo->binfo.resultRowInfo.openWindow); + pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); taosArrayDestroyEx(pInfo->pPrevValues, freeItem); diff --git a/tests/script/tsim/parser/slimit_alter_tags.sim b/tests/script/tsim/parser/slimit_alter_tags.sim index 3827b14b45..b5afbfa56e 100644 --- a/tests/script/tsim/parser/slimit_alter_tags.sim +++ b/tests/script/tsim/parser/slimit_alter_tags.sim @@ -128,6 +128,7 @@ if $rows != 5 then return -1 endi if $data00 != $rowNum then + print expect $rowNum , actual: $data00 return -1 endi if $data10 != $rowNum then From eafad954a1a7057d3fb9db4213ead327d7040ac2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 14:56:57 +0800 Subject: [PATCH 086/102] fix: one little bug --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 42ab6ce3b2..acbdc65f12 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1308,7 +1308,7 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { SBlockData *pBDatal = &pCommitter->dWriter.bDatal; if (pBDatal->suid || pBDatal->uid) { - if (pBDatal->suid != id.suid || id.uid == 0) { + if ((pBDatal->suid != id.suid) || (id.suid == 0)) { if (pBDatal->nRow) { code = tsdbCommitLastBlock(pCommitter); if (code) goto _exit; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 37bd1ccbf6..40841ac85e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -311,17 +311,6 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { if (code) goto _err; } - // sst =========== - tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[0], fname); - if (taosStatFile(fname, &size, NULL)) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - if (size != pSet->aSstF[0]->size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - // sma ============= tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); if (taosStatFile(fname, &size, NULL)) { @@ -335,6 +324,19 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE); if (code) goto _err; } + + // sst =========== + for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { + tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname); + if (taosStatFile(fname, &size, NULL)) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (size != pSet->aSstF[iSst]->size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + } } { From a71be005aea4d8100e7f3bc4e0de936461b00e14 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 15:58:55 +0800 Subject: [PATCH 087/102] fix: another fail CI case --- source/dnode/vnode/src/tsdb/tsdbFS.c | 31 ++++++++++++++++------------ 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 40841ac85e..e6bc9d9936 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -899,21 +899,26 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { pSetOld->aSstF[0]->nRef = 1; } else { for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) { - SSstFile *pSstFile = pSetOld->aSstF[iSst]; - nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); - if (nRef == 0) { - tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); - taosRemoveFile(fname); - taosMemoryFree(pSstFile); - } + if (pSetOld->aSstF[iSst]->commitID != pSetNew->aSstF[iSst]->commitID) { + SSstFile *pSstFile = pSetOld->aSstF[iSst]; + nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); + if (nRef == 0) { + tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pSstFile); + } - pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (pSetOld->aSstF[iSst] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (pSetOld->aSstF[iSst] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst]; + pSetOld->aSstF[iSst]->nRef = 1; + } else { + ASSERT(pSetOld->aSstF[iSst]->size == pSetOld->aSstF[iSst]->size); + ASSERT(pSetOld->aSstF[iSst]->offset == pSetOld->aSstF[iSst]->offset); } - *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst]; - pSetOld->aSstF[iSst]->nRef = 1; } } } else { From 796342dabb6e98d8118a93b25b62e70837e6dfc8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 2 Sep 2022 16:05:22 +0800 Subject: [PATCH 088/102] fix: taosX core dump if schema is null --- include/common/tcommon.h | 1 - source/client/src/taosx.c | 1 + source/dnode/vnode/src/meta/metaQuery.c | 32 +++++++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++-- source/libs/executor/src/executil.c | 1 - source/libs/executor/src/executor.c | 1 - source/libs/executor/src/executorimpl.c | 1 - 7 files changed, 35 insertions(+), 6 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 891c9ab040..37db574d98 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -184,7 +184,6 @@ typedef struct SQueryTableDataCond { STimeWindow twindows; int64_t startVersion; int64_t endVersion; - int64_t schemaVersion; } SQueryTableDataCond; int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); diff --git a/source/client/src/taosx.c b/source/client/src/taosx.c index f016120a1f..cc3a2a198a 100644 --- a/source/client/src/taosx.c +++ b/source/client/src/taosx.c @@ -765,6 +765,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { } taosArrayPush(pRequest->tableList, &pName); + pCreateReq->flags |= TD_CREATE_IF_NOT_EXISTS; // change tag cid to new cid if(pCreateReq->type == TSDB_CHILD_TABLE){ STableMeta* pTableMeta = NULL; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 7df355a59b..c282f17116 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -280,6 +280,38 @@ _query: tDecoderClear(&dc); goto _exit; } + { // Traverse to find the previous qualified data + TBC *pCur; + tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); + STbDbKey key = {.version = sver, .uid = INT64_MAX}; + int c = 0; + tdbTbcMoveTo(pCur, &key, sizeof(key), &c); + if(c < 0){ + tdbTbcMoveToPrev(pCur); + } + + void *pKey = NULL; + void *pVal = NULL; + int vLen = 0, kLen = 0; + while(1){ + int32_t ret = tdbTbcPrev(pCur, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + + STbDbKey *tmp = (STbDbKey*)pKey; + if(tmp->uid != uid){ + continue; + } + SDecoder dcNew = {0}; + SMetaEntry meNew = {0}; + tDecoderInit(&dcNew, pVal, vLen); + metaDecodeEntry(&dcNew, &meNew); + pSchema = tCloneSSchemaWrapper(&meNew.stbEntry.schemaRow); + tDecoderClear(&dcNew); + tdbTbcClose(pCur); + goto _exit; + } + tdbTbcClose(pCur); + } } else if (me.type == TSDB_CHILD_TABLE) { uid = me.ctbEntry.suid; tDecoderClear(&dc); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a92e8189a1..546a20134e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3349,10 +3349,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl } if (pCond->suid != 0) { - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, pCond->schemaVersion); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, pCond->endVersion); } else if (taosArrayGetSize(pTableList) > 0) { STableKeyInfo* pKey = taosArrayGet(pTableList, 0); - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, pCond->schemaVersion); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, pCond->endVersion); } int32_t numOfTables = taosArrayGetSize(pTableList); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 70180d6dc0..03fbda3853 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1300,7 +1300,6 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = -1; - pCond->schemaVersion = -1; // pCond->type = pTableScanNode->scanFlag; int32_t j = 0; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 278f02b228..a242db630d 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -729,7 +729,6 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = sContext->snapVersion; - pCond->schemaVersion = sContext->snapVersion; for (int32_t i = 0; i < pCond->numOfCols; ++i) { pCond->colList[i].type = mtInfo.schema->pSchema[i].type; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b53d35a1a1..fd3450260a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3908,7 +3908,6 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = -1; - pCond->schemaVersion = -1; return TSDB_CODE_SUCCESS; } From 08b36a833da061d1d2ea25cc5f94a86ed3ae4ddb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 16:37:40 +0800 Subject: [PATCH 089/102] disable rbtree test --- source/util/test/trbtreeTest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/util/test/trbtreeTest.cpp b/source/util/test/trbtreeTest.cpp index 518ffed2cf..cabf315df0 100644 --- a/source/util/test/trbtreeTest.cpp +++ b/source/util/test/trbtreeTest.cpp @@ -15,6 +15,7 @@ static int32_t tCmprInteger(const void *p1, const void *p2) { } TEST(trbtreeTest, rbtree_test1) { +#if 0 SRBTree rt; tRBTreeCreate(&rt, tCmprInteger); int a[] = {1, 3, 4, 2, 7, 5, 8}; @@ -35,4 +36,5 @@ TEST(trbtreeTest, rbtree_test1) { // printf("%d\n", la); pNode = tRBTreeIterNext(&rti); } +#endif } \ No newline at end of file From 86348e74392c1b290747eab79471059d1cbd978f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Sep 2022 17:27:47 +0800 Subject: [PATCH 090/102] fix(query): initialize the last block reader successfully. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7f492f73b0..7133ec1717 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -353,12 +353,13 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { int32_t step = asc ? 1 : -1; pIter->index += step; - pIter->pLastBlockReader->uid = 0; - tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { return false; } + pIter->pLastBlockReader->uid = 0; + tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); + // check file the time range of coverage STimeWindow win = {0}; @@ -2161,8 +2162,6 @@ _err: static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; - - initMemDataIterator(pScanInfo, pReader); TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); if (pRow != NULL) { key = TSDBROW_KEY(pRow); @@ -2356,6 +2355,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { pBlock = getCurrentBlock(pBlockIter); } + initLastBlockReader(pLastBlockReader, pScanInfo, pReader); TSDBKEY key = getCurrentKeyInBuf(pScanInfo, pReader); if (pBlockInfo == NULL) { // build data block from last data file From 50efa376914efcdac265580c9c162d9ec2bdab5a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Sep 2022 19:01:26 +0800 Subject: [PATCH 091/102] fix: fix taosd order crash issue --- source/libs/executor/src/tsort.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 168cd21c44..63fc9d9e1c 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -227,9 +227,9 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int continue; } - SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); + int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex); - void* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); + void* pPage = getBufPage(pHandle->pBuf, *pPgId); code = blockDataFromBuf(pSource->src.pBlock, pPage); if (code != TSDB_CODE_SUCCESS) { return code; @@ -302,9 +302,9 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource *pSource, SMultiwayMergeT pSource->pageIndex = -1; pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock); } else { - SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); + int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex); - void* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); + void* pPage = getBufPage(pHandle->pBuf, *pPgId); int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage); if (code != TSDB_CODE_SUCCESS) { return code; From b159d3cdf54a0e355a5235233de369efab5365b4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Sep 2022 10:05:57 +0800 Subject: [PATCH 092/102] fix(query): fix syntax error on windows platform --- include/util/trbtree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 50e2663648..0ca7e3ba95 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -51,7 +51,7 @@ struct SRBTreeNode { SRBTreeNode *parent; SRBTreeNode *left; SRBTreeNode *right; - uint8_t payload[0]; + uint8_t payload[]; }; struct SRBTree { From 4ace5bef9c60830e5564a675c38fdce5c3010276 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Sep 2022 11:08:36 +0800 Subject: [PATCH 093/102] fix(query): fix syntax error. --- include/util/trbtree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 0ca7e3ba95..50e2663648 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -51,7 +51,7 @@ struct SRBTreeNode { SRBTreeNode *parent; SRBTreeNode *left; SRBTreeNode *right; - uint8_t payload[]; + uint8_t payload[0]; }; struct SRBTree { From bbbbebab893ea78233dc8ddbafac36aa730d3b44 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 3 Sep 2022 13:37:48 +0800 Subject: [PATCH 094/102] fix: windows compile --- include/util/trbtree.h | 3 ++- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 11 ++++++----- source/util/src/trbtree.c | 10 +++++----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 50e2663648..f6d37e3d75 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -51,9 +51,10 @@ struct SRBTreeNode { SRBTreeNode *parent; SRBTreeNode *left; SRBTreeNode *right; - uint8_t payload[0]; }; +#define RBTREE_NODE_PAYLOAD(N) ((const void *)&(N)[1]) + struct SRBTree { tRBTreeCmprFn cmprFn; int64_t n; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index ce36d74467..25f57e21fc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -273,7 +273,8 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { } } -int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange) { +int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t uid, + STimeWindow *pTimeWindow, SVersionRange *pVerRange) { pMTree->backward = backward; pMTree->pIter = NULL; pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); @@ -284,8 +285,8 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFRead tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); int32_t code = TSDB_CODE_OUT_OF_MEMORY; - struct SLDataIter* pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; - for(int32_t i = 0; i < pFReader->pSet->nSstF; ++i) { // open all last file + struct SLDataIter *pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; + for (int32_t i = 0; i < pFReader->pSet->nSstF; ++i) { // open all last file code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); if (code != TSDB_CODE_SUCCESS) { goto _end; @@ -302,7 +303,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFRead return code; - _end: +_end: tMergeTreeClose(pMTree); return code; } @@ -322,7 +323,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { // compare with min in RB Tree pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); if (pMTree->pIter && pIter) { - int32_t c = pMTree->rbt.cmprFn(pMTree->pIter->node.payload, &pIter->node.payload); + int32_t c = pMTree->rbt.cmprFn(RBTREE_NODE_PAYLOAD(&pMTree->pIter->node), RBTREE_NODE_PAYLOAD(&pIter->node)); if (c > 0) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); pMTree->pIter = NULL; diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 2d81a6c5b4..65f1bac60a 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -219,7 +219,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { while (temp != pTree->NIL) { y = temp; - int32_t c = pTree->cmprFn(z->payload, temp->payload); + int32_t c = pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(temp)); if (c < 0) { temp = temp->left; } else if (c > 0) { @@ -232,7 +232,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { if (y == pTree->NIL) { pTree->root = z; - } else if (pTree->cmprFn(z->payload, y->payload) < 0) { + } else if (pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(y)) < 0) { y->left = z; } else { y->right = z; @@ -245,10 +245,10 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { tRBTreePutFix(pTree, z); // update min/max node - if (pTree->min == pTree->NIL || pTree->cmprFn(pTree->min->payload, z->payload) > 0) { + if (pTree->min == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->min), RBTREE_NODE_PAYLOAD(z)) > 0) { pTree->min = z; } - if (pTree->max == pTree->NIL || pTree->cmprFn(pTree->max->payload, z->payload) < 0) { + if (pTree->max == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->max), RBTREE_NODE_PAYLOAD(z)) < 0) { pTree->max = z; } pTree->n++; @@ -313,7 +313,7 @@ SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { SRBTreeNode *pNode = pTree->root; while (pNode != pTree->NIL) { - int32_t c = pTree->cmprFn(pKey, pNode->payload); + int32_t c = pTree->cmprFn(pKey, RBTREE_NODE_PAYLOAD(pNode)); if (c < 0) { pNode = pNode->left; From 82f0ba3655f29f07372b983e22071f45b3475dfc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 3 Sep 2022 14:18:29 +0800 Subject: [PATCH 095/102] fix: windows compile error --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 25f57e21fc..b388e5a0fe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -252,8 +252,8 @@ SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } // SMergeTree ================================================= static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { - SLDataIter *pIter1 = (SLDataIter *)(p1 - sizeof(SRBTreeNode)); - SLDataIter *pIter2 = (SLDataIter *)(p2 - sizeof(SRBTreeNode)); + SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - sizeof(SRBTreeNode)); + SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - sizeof(SRBTreeNode)); TSDBKEY key1 = TSDBROW_KEY(&pIter1->rInfo.row); TSDBKEY key2 = TSDBROW_KEY(&pIter2->rInfo.row); From 1c89972fc6c97ef3a0df57e34eecbeeafb91bdc7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Sep 2022 22:00:24 +0800 Subject: [PATCH 096/102] fix(query): add multiple blockdata structure. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 63 ++++++++++++++------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index ce36d74467..95f26161c3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -24,7 +24,8 @@ typedef struct SLDataIter { int8_t backward; SArray *aSstBlk; int32_t iSstBlk; - SBlockData bData; + SBlockData bData[2]; + int32_t loadIndex; int32_t iRow; SRowInfo rInfo; uint64_t uid; @@ -32,6 +33,15 @@ typedef struct SLDataIter { SVersionRange verRange; } SLDataIter; +static SBlockData* getCurrentBlock(SLDataIter* pIter) { + return &pIter->bData[pIter->loadIndex]; +} + +static SBlockData* getNextBlock(SLDataIter* pIter) { + pIter->loadIndex ^= 1; + return getCurrentBlock(pIter); +} + int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iSst, int8_t backward, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange) { int32_t code = 0; @@ -53,7 +63,12 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t goto _exit; } - code = tBlockDataCreate(&(*pIter)->bData); + code = tBlockDataCreate(&(*pIter)->bData[0]); + if (code) { + goto _exit; + } + + code = tBlockDataCreate(&(*pIter)->bData[1]); if (code) { goto _exit; } @@ -95,7 +110,8 @@ _exit: } void tLDataIterClose(SLDataIter *pIter) { - tBlockDataDestroy(&pIter->bData, 1); + tBlockDataDestroy(&pIter->bData[0], 1); + tBlockDataDestroy(&pIter->bData[1], 1); taosArrayDestroy(pIter->aSstBlk); taosMemoryFree(pIter); } @@ -136,24 +152,26 @@ static void findNextValidRow(SLDataIter *pIter) { bool hasVal = false; int32_t i = pIter->iRow; - for (; i < pIter->bData.nRow && i >= 0; i += step) { - if (pIter->bData.aUid != NULL) { + SBlockData* pBlockData = getCurrentBlock(pIter); + + for (; i < pBlockData->nRow && i >= 0; i += step) { + if (pBlockData->aUid != NULL) { if (!pIter->backward) { - if (pIter->bData.aUid[i] < pIter->uid) { + if (pBlockData->aUid[i] < pIter->uid) { continue; - } else if (pIter->bData.aUid[i] > pIter->uid) { + } else if (pBlockData->aUid[i] > pIter->uid) { break; } } else { - if (pIter->bData.aUid[i] > pIter->uid) { + if (pBlockData->aUid[i] > pIter->uid) { continue; - } else if (pIter->bData.aUid[i] < pIter->uid) { + } else if (pBlockData->aUid[i] < pIter->uid) { break; } } } - int64_t ts = pIter->bData.aTSKEY[i]; + int64_t ts = pBlockData->aTSKEY[i]; if (!pIter->backward) { // asc if (ts > pIter->timeWindow.ekey) { // no more data break; @@ -168,7 +186,7 @@ static void findNextValidRow(SLDataIter *pIter) { } } - int64_t ver = pIter->bData.aVersion[i]; + int64_t ver = pBlockData->aVersion[i]; if (ver < pIter->verRange.minVer) { continue; } @@ -203,14 +221,16 @@ bool tLDataIterNextRow(SLDataIter *pIter) { } int32_t iBlockL = pIter->iSstBlk; + SBlockData* pBlockData = getCurrentBlock(pIter); - if (pIter->bData.nRow == 0 && pIter->pSstBlk != NULL) { // current block not loaded yet - code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, &pIter->bData); + if (pBlockData->nRow == 0 && pIter->pSstBlk != NULL) { // current block not loaded yet + pBlockData = getNextBlock(pIter); + code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData); if (code != TSDB_CODE_SUCCESS) { goto _exit; } - pIter->iRow = (pIter->backward) ? pIter->bData.nRow : -1; + pIter->iRow = (pIter->backward) ? pBlockData->nRow : -1; } pIter->iRow += step; @@ -218,7 +238,7 @@ bool tLDataIterNextRow(SLDataIter *pIter) { while (1) { findNextValidRow(pIter); - if (pIter->iRow >= pIter->bData.nRow || pIter->iRow < 0) { + if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) { tLDataIterNextBlock(pIter); if (pIter->pSstBlk == NULL) { // no more data goto _exit; @@ -228,17 +248,18 @@ bool tLDataIterNextRow(SLDataIter *pIter) { } if (iBlockL != pIter->iSstBlk) { - code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, &pIter->bData); + pBlockData = getNextBlock(pIter); + code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData); if (code) { goto _exit; } - pIter->iRow = pIter->backward ? (pIter->bData.nRow - 1) : 0; + pIter->iRow = pIter->backward ? (pBlockData->nRow - 1) : 0; } } - pIter->rInfo.suid = pIter->bData.suid; - pIter->rInfo.uid = pIter->bData.uid; - pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + pIter->rInfo.suid = pBlockData->suid; + pIter->rInfo.uid = pBlockData->uid; + pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow); _exit: if (code != TSDB_CODE_SUCCESS) { @@ -322,7 +343,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { // compare with min in RB Tree pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); if (pMTree->pIter && pIter) { - int32_t c = pMTree->rbt.cmprFn(pMTree->pIter->node.payload, &pIter->node.payload); + int32_t c = pMTree->rbt.cmprFn(pMTree->pIter->node.payload, pIter->node.payload); if (c > 0) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); pMTree->pIter = NULL; From a06053712d3d12a232d46d950ea34b559a2fbcd6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Sep 2022 22:19:25 +0800 Subject: [PATCH 097/102] fix(query): commit --- include/util/trbtree.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 50e2663648..f6d37e3d75 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -51,9 +51,10 @@ struct SRBTreeNode { SRBTreeNode *parent; SRBTreeNode *left; SRBTreeNode *right; - uint8_t payload[0]; }; +#define RBTREE_NODE_PAYLOAD(N) ((const void *)&(N)[1]) + struct SRBTree { tRBTreeCmprFn cmprFn; int64_t n; From 280bb372705d5e10015461c0d5a5c9f647633cd9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Sep 2022 23:06:39 +0800 Subject: [PATCH 098/102] fix(query): reset fileIter when reset tsdb reader. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7133ec1717..fd61935aea 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -337,13 +337,16 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr); return code; } - - SLastBlockReader* pLReader = pIter->pLastBlockReader; - pLReader->order = pReader->order; - pLReader->window = pReader->window; - pLReader->verRange = pReader->verRange; } + SLastBlockReader* pLReader = pIter->pLastBlockReader; + pLReader->order = pReader->order; + pLReader->window = pReader->window; + pLReader->verRange = pReader->verRange; + + pLReader->uid = 0; + tMergeTreeClose(&pLReader->mergeTree); + tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); return TSDB_CODE_SUCCESS; } From 97b802ad77c776e250d46e5cec60d94ae0dfa516 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 4 Sep 2022 09:01:26 +0800 Subject: [PATCH 099/102] fix: compile error --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5ec40d42b5..75bfbadde2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1379,7 +1379,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, bool mergeBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + // SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); STSRow* pTSRow = NULL; From 597ea1c2bf28f93a79ae7a1a38bc2870837ea9c9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 4 Sep 2022 11:58:02 +0800 Subject: [PATCH 100/102] fix(query): add value check. --- source/client/src/clientImpl.c | 7 ++++++- source/common/src/tdatablock.c | 3 +++ source/libs/executor/src/dataDeleter.c | 4 +++- source/libs/executor/src/dataDispatcher.c | 13 ++++++++++++- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index f91ceb3184..5ebc2729f8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1672,7 +1672,12 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int break; } } - if (!needConvert) return TSDB_CODE_SUCCESS; + + if (!needConvert) { + return TSDB_CODE_SUCCESS; + } + + tscDebug("start to convert form json format string"); char* p = (char*)pResultInfo->pData; int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index bcb96dafcc..16b8e55cf7 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2120,6 +2120,7 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ int32_t* rows = (int32_t*)data; *rows = pBlock->info.rows; data += sizeof(int32_t); + ASSERT(*rows > 0); int32_t* cols = (int32_t*)data; *cols = numOfCols; @@ -2183,6 +2184,8 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ *actualLen = *dataLen; *groupId = pBlock->info.groupId; + ASSERT(*dataLen > 0); + uDebug("build data block, actualLen:%d, rows:%d, cols:%d", *dataLen, *rows, *cols); } const char* blockDecode(SSDataBlock* pBlock, const char* pData) { diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 06b7c13fa2..40198615ea 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -168,7 +168,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf)); taosFreeQitem(pBuf); - *pLen = ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->dataLen; + + SDataCacheEntry* pEntry = (SDataCacheEntry*)pDeleter->nextOutput.pData; + *pLen = pEntry->dataLen; *pQueryEnd = pDeleter->queryEnd; qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows); } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 20396046ba..1697ed63fb 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -93,6 +93,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pBuf->useSize = sizeof(SDataCacheEntry); blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); + ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8)); + ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4)); pBuf->useSize += pEntry->dataLen; @@ -170,7 +172,13 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); - *pLen = ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen; + + SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData; + *pLen = pEntry->dataLen; + + ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8)); + ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4)); + *pQueryEnd = pDispatcher->queryEnd; qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows); } @@ -191,6 +199,9 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; + ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8)); + ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4)); + atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); From ead0a97920675a5ef09c267d5c54dbcba9a32534 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 4 Sep 2022 13:21:41 +0800 Subject: [PATCH 101/102] avoid except --- source/libs/transport/src/transComm.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index a4d679b281..dea96fa3ac 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -287,10 +287,10 @@ void transCtxMerge(STransCtx* dst, STransCtx* src) { STransCtxVal* sVal = (STransCtxVal*)iter; key = taosHashGetKey(sVal, &klen); - STransCtxVal* dVal = taosHashGet(dst->args, key, klen); - if (dVal) { - dst->freeFunc(dVal->val); - } + // STransCtxVal* dVal = taosHashGet(dst->args, key, klen); + // if (dVal) { + // dst->freeFunc(dVal->val); + // } taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); iter = taosHashIterate(src->args, iter); } From 69cc47e62203f3694f302370497b78433884f008 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 4 Sep 2022 18:14:39 +0800 Subject: [PATCH 102/102] test: comment out sml case --- tests/system-test/fulltest.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 4305ceff56..11df13d451 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -156,8 +156,8 @@ python3 ./test.py -f 2-query/sin.py python3 ./test.py -f 2-query/sin.py -R python3 ./test.py -f 2-query/smaTest.py python3 ./test.py -f 2-query/smaTest.py -R -python3 ./test.py -f 2-query/sml.py -python3 ./test.py -f 2-query/sml.py -R +#python3 ./test.py -f 2-query/sml.py +#python3 ./test.py -f 2-query/sml.py -R python3 ./test.py -f 2-query/spread.py python3 ./test.py -f 2-query/spread.py -R python3 ./test.py -f 2-query/sqrt.py @@ -512,6 +512,6 @@ python3 ./test.py -f 2-query/count_partition.py -Q 3 python3 ./test.py -f 2-query/max_partition.py -Q 3 python3 ./test.py -f 2-query/last_row.py -Q 3 python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 -python3 ./test.py -f 2-query/sml.py -Q 3 +#python3 ./test.py -f 2-query/sml.py -Q 3 python3 ./test.py -f 2-query/interp.py -Q 3