From 0dfbfc7cc28071647c7f480eb56da7cf7ec9c0fb Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 23 May 2022 23:55:15 +0800 Subject: [PATCH 1/2] enh: add debug log of tsdb load block data cols --- include/common/taosdef.h | 1 + source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 131 +++++++++++++++++++-- 2 files changed, 121 insertions(+), 11 deletions(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 1d32c9825f..1f3a4d3323 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -96,6 +96,7 @@ extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 #undef TD_DEBUG_PRINT_ROW +#undef TD_DEBUG_PRINT_TSDB_READ_DCOLS #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 0773805684..5426b99730 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -252,6 +252,45 @@ static FORCE_INLINE void tsdbSwapDataCols(SDataCols *pDest, SDataCols *pSrc) { pSrc->cols = pCols; } +static void printTsdbLoadBlkData(SReadH *readh, SDataCols *pDCols, SBlock *pBlock, const char *tag, int32_t ln) { + printf("%s:%d:%" PRIi64 " ================\n", tag, ln, taosGetSelfPthreadId()); + if (pBlock) { + SDFile *pHeadf = TSDB_READ_HEAD_FILE(readh); + printf("%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len, + pHeadf->f.aname); + SDFile *pDFile = pBlock->last ? TSDB_READ_LAST_FILE(readh) : TSDB_READ_DATA_FILE(readh); + printf("%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len, + pDFile->f.aname); + } + SDataCol *pDCol = pDCols->cols + 0; + if (TSKEY_MIN == *(int64_t *)pDCol->pData) { + ASSERT(0); + } + + int rows = pDCols->numOfRows; + for (int r = 0; r < rows; ++r) { + if (pBlock) { + printf("%s:%d:%" PRIi64 ":%p:%d rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len, + rows, r); + } else { + printf("%s:%d:%" PRIi64 ":%s rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), "=== merge === ", rows, r); + } + + int nDataCols = pDCols->numOfCols; + int j = 0; + SCellVal sVal = {0}; + while (j < nDataCols) { + SDataCol *pDataCol = pDCols->cols + j; + tdGetColDataOfRow(&sVal, pDataCol, r, pDCols->bitmapMode); + tdSCellValPrint(&sVal, pDataCol->type); + ++j; + } + printf("\n"); + } + + fflush(stdout); +} + int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { ASSERT(pBlock->numOfSubBlocks > 0); STsdbCfg *pCfg = REPO_CFG(pReadh->pRepo); @@ -266,15 +305,23 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { } } - if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0], TSDB_BITMODE_ONE_BIT) < 0) return -1; +#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS + printTsdbLoadBlkData(pReadh, pReadh->pDCols[0], iBlock, __func__, __LINE__); +#endif for (int i = 1; i < pBlock->numOfSubBlocks; i++) { iBlock++; if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1], TSDB_BITMODE_DEFAULT) < 0) return -1; +#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS + printTsdbLoadBlkData(pReadh, pReadh->pDCols[1], iBlock, __func__, __LINE__); +#endif // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) return -1; +#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS + printTsdbLoadBlkData(pReadh, pReadh->pDCols[0], iBlock, " === MERGE === ", __LINE__); +#endif } // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line if (pBlock->numOfSubBlocks == 1) { @@ -286,6 +333,9 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { } tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]); ASSERT(pReadh->pDCols[0]->bitmapMode != 0); +#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS + printTsdbLoadBlkData(pReadh, pReadh->pDCols[0], iBlock, " === UPDATE FILTER === ", __LINE__); +#endif } ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); @@ -295,6 +345,53 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { return 0; } +static void printTsdbLoadBlkDataCols(SReadH *readh, SDataCols *pDCols, SBlock *pBlock, const int16_t *colIds, + int numOfColsIds, const char *tag, int32_t ln) { + printf("ausp:%s:%d:%" PRIi64 " ================\n", tag, ln, taosGetSelfPthreadId()); + if (pBlock) { + SDFile *pHeadf = TSDB_READ_HEAD_FILE(readh); + printf("ausp:%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len, + pHeadf->f.aname); + SDFile *pDFile = pBlock->last ? TSDB_READ_LAST_FILE(readh) : TSDB_READ_DATA_FILE(readh); + printf("ausp:%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len, + pDFile->f.aname); + } + + int rows = pDCols->numOfRows; + for (int r = 0; r < rows; ++r) { + if (pBlock) { + printf("ausp:%s:%d:%" PRIi64 ":%p:%d rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), pBlock, + (int32_t)pBlock->len, rows, r); + } else { + printf("ausp:%s:%d:%" PRIi64 ":%s rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), "=== merge === ", rows, r); + } + + int nDataCols = pDCols->numOfCols; + int j = 0, k = 0; + SCellVal sVal = {0}; + while (j < nDataCols) { + if (k >= numOfColsIds) break; + SDataCol *pDataCol = pDCols->cols + j; + int16_t colId1 = pDataCol->colId; + int16_t colId2 = *(colIds + k); + if (colId1 < colId2) { + ++j; + } else if (colId1 > colId2) { + ++k; // colId2 not exists in SDataCols + printf("NotExists "); + } else { + tdGetColDataOfRow(&sVal, pDataCol, r, pDCols->bitmapMode); + tdSCellValPrint(&sVal, pDataCol->type); + ++j; + ++k; + } + } + printf("\n"); + } + + fflush(stdout); +} + // TODO: filter by Multi-Version int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds, bool mergeBitmap) { @@ -310,14 +407,25 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, } } - if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds, TSDB_BITMODE_ONE_BIT) < 0) return -1; + if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds, TSDB_BITMODE_ONE_BIT) < 0) + return -1; +#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS + printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], iBlock, colIds, numOfColsIds, __func__, __LINE__); +#endif for (int i = 1; i < pBlock->numOfSubBlocks; i++) { iBlock++; - if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds, TSDB_BITMODE_DEFAULT) < 0) return -1; + if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds, TSDB_BITMODE_DEFAULT) < 0) + return -1; +#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS + printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[1], iBlock, colIds, numOfColsIds, __func__, __LINE__); +#endif // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) return -1; +#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS + printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], NULL, colIds, numOfColsIds, __func__, __LINE__); +#endif } // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line if (pBlock->numOfSubBlocks == 1) { @@ -329,18 +437,23 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, } tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]); ASSERT(pReadh->pDCols[0]->bitmapMode != 0); +#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS + printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], NULL, colIds, numOfColsIds, + " === update filter === ", __LINE__); +#endif } if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { for (int i = 0; i < numOfColsIds; ++i) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; if (pDataCol->len > 0 && pDataCol->bitmap) { - ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID); - ASSERT(pDataCol->pBitmap); tdMergeBitmap(pDataCol->pBitmap, pReadh->pDCols[0]->numOfRows, pDataCol->pBitmap); tdDataColsSetBitmapI(pReadh->pDCols[0]); } } +#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS + printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], NULL, colIds, numOfColsIds, " === merge bitmap === ", __LINE__); +#endif } ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); @@ -551,9 +664,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat tdResetDataCols(pDataCols); - if (tdIsBitmapModeI(bitmapMode)) { - tdDataColsSetBitmapI(pDataCols); - } + pDataCols->bitmapMode = bitmapMode; if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlock->len) < 0) return -1; @@ -740,9 +851,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * tdResetDataCols(pDataCols); - if (tdIsBitmapModeI(bitmapMode)) { - tdDataColsSetBitmapI(pDataCols); - } + pDataCols->bitmapMode = bitmapMode; // If only load timestamp column, no need to load SBlockData part if (numOfColIds > 1 && tsdbLoadBlockOffset(pReadh, pBlock) < 0) return -1; From 4a110b0170f2544f759ab31752077a430d9c842d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 23 May 2022 23:59:16 +0800 Subject: [PATCH 2/2] enh: code optimization --- include/common/taosdef.h | 2 +- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 30 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 1f3a4d3323..d39c7a1215 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -96,7 +96,7 @@ extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 #undef TD_DEBUG_PRINT_ROW -#undef TD_DEBUG_PRINT_TSDB_READ_DCOLS +#undef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 5426b99730..f66037b16d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -306,20 +306,20 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { } if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0], TSDB_BITMODE_ONE_BIT) < 0) return -1; -#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS +#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS printTsdbLoadBlkData(pReadh, pReadh->pDCols[0], iBlock, __func__, __LINE__); #endif for (int i = 1; i < pBlock->numOfSubBlocks; i++) { iBlock++; if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1], TSDB_BITMODE_DEFAULT) < 0) return -1; -#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS +#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS printTsdbLoadBlkData(pReadh, pReadh->pDCols[1], iBlock, __func__, __LINE__); #endif // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) return -1; -#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS +#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS printTsdbLoadBlkData(pReadh, pReadh->pDCols[0], iBlock, " === MERGE === ", __LINE__); #endif } @@ -333,7 +333,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { } tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]); ASSERT(pReadh->pDCols[0]->bitmapMode != 0); -#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS +#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS printTsdbLoadBlkData(pReadh, pReadh->pDCols[0], iBlock, " === UPDATE FILTER === ", __LINE__); #endif } @@ -347,23 +347,23 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { static void printTsdbLoadBlkDataCols(SReadH *readh, SDataCols *pDCols, SBlock *pBlock, const int16_t *colIds, int numOfColsIds, const char *tag, int32_t ln) { - printf("ausp:%s:%d:%" PRIi64 " ================\n", tag, ln, taosGetSelfPthreadId()); + printf("%s:%d:%" PRIi64 " ================\n", tag, ln, taosGetSelfPthreadId()); if (pBlock) { SDFile *pHeadf = TSDB_READ_HEAD_FILE(readh); - printf("ausp:%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len, + printf("%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len, pHeadf->f.aname); SDFile *pDFile = pBlock->last ? TSDB_READ_LAST_FILE(readh) : TSDB_READ_DATA_FILE(readh); - printf("ausp:%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len, + printf("%s:%d:%" PRIi64 ":%p:%d %s\n", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len, pDFile->f.aname); } int rows = pDCols->numOfRows; for (int r = 0; r < rows; ++r) { if (pBlock) { - printf("ausp:%s:%d:%" PRIi64 ":%p:%d rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), pBlock, - (int32_t)pBlock->len, rows, r); + printf("%s:%d:%" PRIi64 ":%p:%d rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), pBlock, (int32_t)pBlock->len, + rows, r); } else { - printf("ausp:%s:%d:%" PRIi64 ":%s rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), "=== merge === ", rows, r); + printf("%s:%d:%" PRIi64 ":%s rows[%d][%d] ", tag, ln, taosGetSelfPthreadId(), "=== merge === ", rows, r); } int nDataCols = pDCols->numOfCols; @@ -409,21 +409,21 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds, TSDB_BITMODE_ONE_BIT) < 0) return -1; -#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS +#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], iBlock, colIds, numOfColsIds, __func__, __LINE__); #endif for (int i = 1; i < pBlock->numOfSubBlocks; i++) { iBlock++; if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds, TSDB_BITMODE_DEFAULT) < 0) return -1; -#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS +#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[1], iBlock, colIds, numOfColsIds, __func__, __LINE__); #endif // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) return -1; -#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS +#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], NULL, colIds, numOfColsIds, __func__, __LINE__); #endif } @@ -437,7 +437,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, } tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]); ASSERT(pReadh->pDCols[0]->bitmapMode != 0); -#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS +#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], NULL, colIds, numOfColsIds, " === update filter === ", __LINE__); #endif @@ -451,7 +451,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, tdDataColsSetBitmapI(pReadh->pDCols[0]); } } -#ifdef TD_DEBUG_PRINT_TSDB_READ_DCOLS +#ifdef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS printTsdbLoadBlkDataCols(pReadh, pReadh->pDCols[0], NULL, colIds, numOfColsIds, " === merge bitmap === ", __LINE__); #endif }