From e2d4203be0ad864c6bb86ed3663c0919fbddf930 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 21 Mar 2023 18:44:55 +0800 Subject: [PATCH 1/7] test: change lower assert to system assert --- tests/system-test/0-others/check_assert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/check_assert.py b/tests/system-test/0-others/check_assert.py index 59fb223528..ff69b9eeec 100644 --- a/tests/system-test/0-others/check_assert.py +++ b/tests/system-test/0-others/check_assert.py @@ -28,10 +28,10 @@ import os NO_FOUND = 0 # not found assert or ASSERT FOUND_OK = 1 # found ASSERT and valid usage FOUND_NOIF = 2 # found ASSERT but no if like ASSERT(...) -FOUND_LOWER = 3 # found assert write with lower letters +FOUND_LOWER = 3 # found assert write with system assert FOUND_HAVENOT = 4 # found ASSERT have if but have not like if(!ASSERT) -code_strs = ["not found", "valid", "found but no if", "lower assert","found but have not"] +code_strs = ["not found", "valid", "found but no if", "system assert","found but have not"] # From 0b31aac2c5c8c40b2656ea52c66d517922c48748 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 22 Mar 2023 10:19:58 +0800 Subject: [PATCH 2/7] enh(tsdb/cache): skip data block with merge tree --- source/dnode/vnode/src/inc/tsdb.h | 2 + source/dnode/vnode/src/tsdb/tsdbCache.c | 94 ++++++--------------- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 27 +++++- 3 files changed, 55 insertions(+), 68 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0b38ce6d24..452b1f6c0b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -687,6 +687,8 @@ typedef struct SSttBlockLoadInfo { STSchema *pSchema; int16_t *colIds; int32_t numOfCols; + bool checkRemainingRow; + bool isLast; bool sttBlockLoaded; int32_t numOfStt; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3d01184e78..7c45a7ce82 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -590,6 +590,7 @@ typedef struct { SDataFReader **pDataFReader; TSDBROW row; + bool checkRemainingRow; SMergeTree mergeTree; SMergeTree *pMergeTree; SSttBlockLoadInfo *pLoadInfo; @@ -600,7 +601,6 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa int nCols) { SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; int32_t code = 0; - bool checkRemainingRow = true; switch (state->state) { case SFSLASTNEXTROW_FS: @@ -633,8 +633,10 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa if (code) goto _err; } - state->pLoadInfo->colIds = aCols; - state->pLoadInfo->numOfCols = nCols; + for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { + state->pLoadInfo[i].colIds = aCols; + state->pLoadInfo[i].numOfCols = nCols; + } tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid, &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX}, &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true); @@ -650,75 +652,33 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa goto _next_fileset; } state->state = SFSLASTNEXTROW_BLOCKROW; - checkRemainingRow = false; } case SFSLASTNEXTROW_BLOCKROW: { - bool skipRow = false; - do { - bool hasVal = false; - state->row = tMergeTreeGetRow(&state->mergeTree); - *ppRow = &state->row; - if (nCols != state->pLoadInfo->numOfCols) { - state->pLoadInfo->numOfCols = nCols; - } - hasVal = tMergeTreeNext(&state->mergeTree); - if (TSDBROW_TS(&state->row) <= state->lastTs) { - *pIgnoreEarlierTs = true; - *ppRow = NULL; - return code; + bool hasVal = false; + state->row = tMergeTreeGetRow(&state->mergeTree); + *ppRow = &state->row; + if (nCols != state->pLoadInfo->numOfCols) { + for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { + state->pLoadInfo[i].numOfCols = nCols; } + } + state->pLoadInfo->checkRemainingRow = state->checkRemainingRow; + state->pLoadInfo->isLast = isLast; + hasVal = tMergeTreeNext(&state->mergeTree); + if (TSDBROW_TS(&state->row) <= state->lastTs) { + *pIgnoreEarlierTs = true; + *ppRow = NULL; + return code; + } - *pIgnoreEarlierTs = false; - if (!hasVal) { - state->state = SFSLASTNEXTROW_FILESET; - break; - } - - if (checkRemainingRow) { - bool skipBlock = true; - - SBlockData *pBlockData = state->row.pBlockData; - - for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) { - for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) { - SColData *pColData = &pBlockData->aColData[colIndex]; - int16_t cid = pColData->cid; - - if (cid == aCols[inputColIndex]) { - if (isLast && (pColData->flag & HAS_VALUE)) { - skipBlock = false; - break; - } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { - skipBlock = false; - break; - } - } - } - } - /* - for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) { - SColData *pColData = &pBlockData->aColData[colIndex]; - int16_t cid = pColData->cid; - - if (inputColIndex < nCols && cid == aCols[inputColIndex]) { - if (isLast && (pColData->flag & HAS_VALUE)) { - skipBlock = false; - break; - } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { - skipBlock = false; - break; - } - - ++inputColIndex; - } - } - */ - if (skipBlock) { - skipRow = true; - } - } - } while (skipRow); + *pIgnoreEarlierTs = false; + if (!hasVal) { + state->state = SFSLASTNEXTROW_FILESET; + } + if (!state->checkRemainingRow) { + state->checkRemainingRow = true; + } return code; } default: diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 943b16116c..fa8870835c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -504,9 +504,34 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { pIter->iRow += step; while (1) { + bool skipBlock = false; + findNextValidRow(pIter, idStr); - if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) { + if (pIter->pBlockLoadInfo->checkRemainingRow) { + skipBlock = true; + int16_t *aCols = pIter->pBlockLoadInfo->colIds; + int nCols = pIter->pBlockLoadInfo->numOfCols; + bool isLast = pIter->pBlockLoadInfo->isLast; + for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) { + for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) { + SColData *pColData = &pBlockData->aColData[colIndex]; + int16_t cid = pColData->cid; + + if (cid == aCols[inputColIndex]) { + if (isLast && (pColData->flag & HAS_VALUE)) { + skipBlock = false; + break; + } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + skipBlock = false; + break; + } + } + } + } + } + + if (skipBlock || pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) { tLDataIterNextBlock(pIter, idStr); if (pIter->pSttBlk == NULL) { // no more data goto _exit; From cb93ac26138771724ef528a683e7b18079e21c8e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 22 Mar 2023 10:50:34 +0800 Subject: [PATCH 3/7] enh(tsdb/cache): make merge tree next one less --- source/dnode/vnode/src/tsdb/tsdbCache.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 7c45a7ce82..e78127e195 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -641,6 +641,17 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX}, &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true); state->pMergeTree = &state->mergeTree; + state->state = SFSLASTNEXTROW_BLOCKROW; + } + case SFSLASTNEXTROW_BLOCKROW: { + if (nCols != state->pLoadInfo->numOfCols) { + for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { + state->pLoadInfo[i].numOfCols = nCols; + } + + state->pLoadInfo->checkRemainingRow = state->checkRemainingRow; + state->pLoadInfo->isLast = isLast; + } bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { if (tMergeTreeIgnoreEarlierTs(&state->mergeTree)) { @@ -651,20 +662,9 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa state->state = SFSLASTNEXTROW_FILESET; goto _next_fileset; } - state->state = SFSLASTNEXTROW_BLOCKROW; - } - case SFSLASTNEXTROW_BLOCKROW: { - bool hasVal = false; state->row = tMergeTreeGetRow(&state->mergeTree); *ppRow = &state->row; - if (nCols != state->pLoadInfo->numOfCols) { - for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { - state->pLoadInfo[i].numOfCols = nCols; - } - } - state->pLoadInfo->checkRemainingRow = state->checkRemainingRow; - state->pLoadInfo->isLast = isLast; - hasVal = tMergeTreeNext(&state->mergeTree); + if (TSDBROW_TS(&state->row) <= state->lastTs) { *pIgnoreEarlierTs = true; *ppRow = NULL; From 0bf98096c907605ed6f8fa9656c3321234aa4904 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 22 Mar 2023 10:57:27 +0800 Subject: [PATCH 4/7] fix(tsdb/cache): init isLast once for each fileset --- source/dnode/vnode/src/tsdb/tsdbCache.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index e78127e195..6fc8ad8be6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -636,6 +636,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { state->pLoadInfo[i].colIds = aCols; state->pLoadInfo[i].numOfCols = nCols; + state->pLoadInfo[i].isLast = isLast; } tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid, &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX}, @@ -647,10 +648,9 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa if (nCols != state->pLoadInfo->numOfCols) { for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { state->pLoadInfo[i].numOfCols = nCols; - } - state->pLoadInfo->checkRemainingRow = state->checkRemainingRow; - state->pLoadInfo->isLast = isLast; + state->pLoadInfo[i].checkRemainingRow = state->checkRemainingRow; + } } bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { From 5d641a8934652ba388308502a82f58258e9fd3b2 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 23 Mar 2023 17:55:53 +0800 Subject: [PATCH 5/7] fix(tdb/restore): restore tdb journal backward --- source/libs/tdb/src/db/tdbPager.c | 35 ++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 29b7fa740c..da5b37cdce 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -947,6 +947,12 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { return 0; } +static int32_t txnIdCompareDesc(const void *pLeft, const void *pRight) { + int64_t lhs = *(int64_t *)pLeft; + int64_t rhs = *(int64_t *)pRight; + return lhs > rhs ? -1 : 1; +} + int tdbPagerRestoreJournals(SPager *pPager) { tdbDirEntryPtr pDirEntry; tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); @@ -955,20 +961,29 @@ int tdbPagerRestoreJournals(SPager *pPager) { return -1; } + SArray *pTxnList = taosArrayInit(16, sizeof(int64_t)); + while ((pDirEntry = tdbReadDir(pDir)) != NULL) { char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { - char jname[TD_PATH_MAX] = {0}; - int dirLen = strlen(pPager->pEnv->dbName); - memcpy(jname, pPager->pEnv->dbName, dirLen); - jname[dirLen] = '/'; - memcpy(jname + dirLen + 1, name, strlen(name)); - if (tdbPagerRestore(pPager, jname) < 0) { - tdbCloseDir(&pDir); + int64_t txnId = -1; + sscanf(name, TDB_MAINDB_NAME "-journal.%" PRId64, &txnId); + taosArrayPush(pTxnList, &txnId); + } + } + taosArraySort(pTxnList, txnIdCompareDesc); + for (int i = 0; i < TARRAY_SIZE(pTxnList); ++i) { + int64_t *pTxnId = taosArrayGet(pTxnList, i); + char jname[TD_PATH_MAX] = {0}; + int dirLen = strlen(pPager->pEnv->dbName); + memcpy(jname, pPager->pEnv->dbName, dirLen); + jname[dirLen] = '/'; + sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); + if (tdbPagerRestore(pPager, jname) < 0) { + tdbCloseDir(&pDir); - tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), name); - return -1; - } + tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname); + return -1; } } From a07366ed02ffec077e7f80c037b44f86276e3c59 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 23 Mar 2023 18:23:14 +0800 Subject: [PATCH 6/7] fix(tdb/pager): destroy txn id list --- source/libs/tdb/src/db/tdbPager.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index da5b37cdce..5ea9be63db 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -987,6 +987,7 @@ int tdbPagerRestoreJournals(SPager *pPager) { } } + taosArrayDestroy(pTxnList); tdbCloseDir(&pDir); return 0; From fb883786f4f05ae08e4db9cb9dbc71e368b847b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Mar 2023 10:06:09 +0800 Subject: [PATCH 7/7] chore(deps): bump spring-core in /examples/JDBC/taosdemo (#20610) Bumps [spring-core](https://github.com/spring-projects/spring-framework) from 5.3.20 to 5.3.26. - [Release notes](https://github.com/spring-projects/spring-framework/releases) - [Commits](https://github.com/spring-projects/spring-framework/compare/v5.3.20...v5.3.26) --- updated-dependencies: - dependency-name: org.springframework:spring-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/JDBC/taosdemo/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/JDBC/taosdemo/pom.xml b/examples/JDBC/taosdemo/pom.xml index 68224bbad5..4731d8e237 100644 --- a/examples/JDBC/taosdemo/pom.xml +++ b/examples/JDBC/taosdemo/pom.xml @@ -10,7 +10,7 @@ Demo project for TDengine - 5.3.20 + 5.3.26