From 14ddf539e04c3e095516e30ce56052a02a0b7140 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 17:21:04 +0800 Subject: [PATCH 01/55] refactor(query): do some internal refactor. --- include/common/tcommon.h | 7 +- include/common/tdatablock.h | 11 +- include/common/tmsg.h | 5 - include/libs/function/function.h | 17 - source/common/src/tdatablock.c | 303 +++++++++++------- source/common/test/commonTests.cpp | 46 +-- source/dnode/mnode/impl/src/mndShow.c | 3 +- source/dnode/vnode/inc/vnode.h | 4 +- source/dnode/vnode/src/tq/tqExec.c | 9 +- source/dnode/vnode/src/tq/tqRead.c | 48 +-- source/dnode/vnode/src/tq/tqSink.c | 4 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 69 ++-- source/dnode/vnode/test/tsdbSmaTest.cpp | 2 +- source/libs/command/src/command.c | 30 +- source/libs/command/src/explain.c | 17 +- source/libs/executor/src/dataDeleter.c | 2 +- source/libs/executor/src/executil.c | 18 +- source/libs/executor/src/executor.c | 2 + source/libs/executor/src/executorimpl.c | 54 ++-- source/libs/executor/src/groupoperator.c | 4 +- source/libs/executor/src/scanoperator.c | 53 ++- source/libs/executor/src/sortoperator.c | 7 +- source/libs/executor/src/tfill.c | 2 +- source/libs/executor/src/timewindowoperator.c | 13 +- source/libs/executor/src/tsort.c | 8 +- source/libs/executor/test/executorTests.cpp | 42 +-- source/libs/executor/test/sortTests.cpp | 24 +- source/libs/function/inc/taggfunction.h | 13 - source/libs/function/src/builtinsimpl.c | 16 +- source/libs/function/src/tudf.c | 43 +-- source/libs/function/test/runUdf.c | 50 ++- .../libs/index/test/index_executor_tests.cpp | 2 +- source/libs/scalar/inc/sclInt.h | 2 +- source/libs/scalar/src/filter.c | 2 +- source/libs/scalar/src/scalar.c | 22 +- source/libs/scalar/src/sclfunc.c | 4 +- source/libs/scalar/src/sclvector.c | 4 +- .../libs/scalar/test/filter/filterTests.cpp | 282 ++++++++-------- .../libs/scalar/test/scalar/scalarTests.cpp | 60 ++-- source/libs/stream/src/streamDispatch.c | 6 +- 40 files changed, 585 insertions(+), 725 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a05287761e..371f182d53 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -66,11 +66,10 @@ typedef struct SDataBlockInfo { int32_t rows; int32_t rowSize; int64_t uid; // the uid of table, from which current data block comes - int64_t blockId; // block id, generated by physical planner + uint16_t blockId; // block id, generated by physical planner uint64_t groupId; // no need to serialize - int16_t numOfCols; int16_t hasVarCol; - int32_t capacity; + uint32_t capacity; // TODO: optimize and remove following int32_t childId; // used for stream, do not serialize EStreamType type; // used for stream, do not serialize @@ -91,7 +90,7 @@ typedef struct SVarColAttr { // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // TODO filter info needs to be removed + SColumnInfo info; // column info bool hasNull; // if current column data has null value. char* pData; // the corresponding block data in memory union { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 709462a744..d65fdc0675 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -185,7 +185,7 @@ int32_t getJsonValueLen(const char* data); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); -int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); @@ -211,7 +211,7 @@ size_t blockDataGetSerialMetaSize(uint32_t numOfCols); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); -int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows); +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); @@ -222,6 +222,11 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); +SSDataBlock* createDataBlock(); +int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); + +SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); +SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index); void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); @@ -235,7 +240,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { - return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock); + return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock); } static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, diff --git a/include/common/tmsg.h b/include/common/tmsg.h index fd9ff3eb65..a84bc3cd03 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -628,11 +628,6 @@ typedef struct { uint8_t scale; } SColumnInfo; -typedef struct { - int64_t uid; - TSKEY key; // last accessed ts, for subscription -} STableIdInfo; - typedef struct STimeWindow { TSKEY skey; TSKEY ekey; diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 0d8ae289cc..c5842559f8 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -172,23 +172,6 @@ typedef struct tExprNode { void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); -typedef struct SAggFunctionInfo { - char name[FUNCTIONS_NAME_MAX_LENGTH]; - int8_t type; // Scalar function or aggregation function - uint32_t functionId; // Function Id - int8_t sFunctionId; // Transfer function for super table query - uint16_t status; - - bool (*init)(SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*addInput)(SqlFunctionCtx *pCtx); - - // finalizer must be called after all exec has been executed to generated final result. - void (*finalize)(SqlFunctionCtx *pCtx); - void (*combine)(SqlFunctionCtx *pCtx); - - int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); -} SAggFunctionInfo; - struct SScalarParam { SColumnInfoData *columnData; SHashObj *pHashFilter; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5594ee9bcf..7856f3a29b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -293,53 +293,61 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in return numOfRow1 + numOfRow2; } -int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows) { +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRows <= 0) { return numOfRows; } + ASSERT(pBlockInfo->capacity >= numOfRows); + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - // Handle the bitmap - char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); - if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } +// if (pColumnInfoData->capacity < numOfRows) { +// char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); +// if (p == NULL) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// +// memset(p, 0, sizeof(int32_t)); +// pColumnInfoData->capacity = numOfRows; +// pColumnInfoData->varmeta.offset = (int32_t*)p; +// } - pColumnInfoData->varmeta.offset = (int32_t*)p; memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); - - if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { - char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pColumnInfoData->pData = tmp; - pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; - } +// if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { +// char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); +// if (tmp == NULL) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// +// pColumnInfoData->pData = tmp; +// pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; +// } memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); pColumnInfoData->varmeta.length = pSource->varmeta.length; } else { - char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } +// if (pColumnInfoData->capacity < numOfRows) { +// char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); +// if (tmp == NULL) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// +// memset(tmp, 0, BitmapLen(numOfRows)); +// pColumnInfoData->nullbitmap = tmp; +// +// int32_t newSize = numOfRows * pColumnInfoData->info.bytes; +// tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); +// if (tmp == NULL) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// +// pColumnInfoData->pData = tmp; +// pColumnInfoData->capacity = numOfRows; +// } - pColumnInfoData->nullbitmap = tmp; memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); - - if (pSource->pData) { - int32_t newSize = numOfRows * pColumnInfoData->info.bytes; - tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pColumnInfoData->pData = tmp; - memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); - } + memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); } pColumnInfoData->hasNull = pSource->hasNull; @@ -348,8 +356,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p } size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { - ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock)); - return pBlock->info.numOfCols; + return taosArrayGetSize(pBlock->pDataBlock); } size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } @@ -359,7 +366,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) return 0; } - if (pDataBlock->info.numOfCols <= 0) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + if (numOfCols <= 0) { return -1; } @@ -383,7 +391,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { assert(pSrc != NULL && pDest != NULL); int32_t capacity = pDest->info.capacity; - for (int32_t i = 0; i < pDest->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDest->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); @@ -399,9 +408,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { size_t blockDataGetSize(const SSDataBlock* pBlock) { assert(pBlock != NULL); - size_t total = 0; - int32_t numOfCols = pBlock->info.numOfCols; - + size_t total = 0; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); total += colDataGetFullLength(pColInfoData, pBlock->info.rows); @@ -416,7 +424,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd int32_t pageSize) { ASSERT(pBlock != NULL && stopIndex != NULL); - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; int32_t bitmapChar = 1; @@ -477,33 +485,24 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 return NULL; } - SSDataBlock* pDst = taosMemoryCalloc(1, sizeof(SSDataBlock)); + SSDataBlock* pDst = createDataBlock(); if (pDst == NULL) { return NULL; } pDst->info = pBlock->info; - pDst->info.rows = 0; - pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i); colInfo.info = pSrcCol->info; - - if (IS_VAR_DATA_TYPE(pSrcCol->info.type)) { - SVarColAttr* pAttr = &colInfo.varmeta; - pAttr->offset = taosMemoryCalloc(rowCount, sizeof(int32_t)); - } else { - colInfo.nullbitmap = taosMemoryCalloc(1, BitmapLen(rowCount)); - colInfo.pData = taosMemoryCalloc(rowCount, colInfo.info.bytes); - } - - taosArrayPush(pDst->pDataBlock, &colInfo); + blockDataAppendColInfo(pDst, &colInfo); } - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + blockDataEnsureCapacity(pDst, rowCount); + + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); @@ -541,7 +540,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { // write the number of rows *(uint32_t*)buf = pBlock->info.rows; - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; char* pStart = buf + sizeof(uint32_t); @@ -571,7 +570,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { pBlock->info.rows = *(int32_t*)buf; - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); const char* pStart = buf + sizeof(uint32_t); for (int32_t i = 0; i < numOfCols; ++i) { @@ -620,7 +619,8 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) pBlock->info.rows = *(int32_t*)buf; pBlock->info.groupId = *(uint64_t*)(buf + sizeof(int32_t)); - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + const char* pStart = buf + sizeof(uint32_t) + sizeof(uint64_t); for (int32_t i = 0; i < numOfCols; ++i) { @@ -666,7 +666,7 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { if (pBlock->info.rowSize == 0) { size_t rowSize = 0; - size_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); rowSize += pColInfo->info.bytes; @@ -693,7 +693,7 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { ASSERT(pBlock != NULL); double rowSize = 0; - size_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); rowSize += pColInfo->info.bytes; @@ -762,7 +762,7 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, int32_t tupleIndex) { int32_t code = 0; - int32_t numOfCols = pSrcBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = &pDstCols[i]; @@ -785,7 +785,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co return TSDB_CODE_SUCCESS; } -static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, int32_t* index) { +static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) { #if 0 for (int32_t i = 0; i < pDataBlock->info.rows; ++i) { int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]); @@ -794,7 +794,8 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB } } #else - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = &pCols[i]; SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); @@ -821,7 +822,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { int32_t rows = pDataBlock->info.rows; - int32_t numOfCols = pDataBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData)); if (pCols == NULL) { @@ -848,7 +849,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { } static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { - int32_t numOfCols = pDataBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); @@ -909,8 +910,10 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { } } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + if (taosArrayGetSize(pOrderInfo) == 1 && (!sortColumnHasNull)) { - if (pDataBlock->info.numOfCols == 1) { + if (numOfCols == 1) { if (!varTypeSort) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0); SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0); @@ -926,7 +929,8 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { return TSDB_CODE_SUCCESS; } else { // var data type } - } else if (pDataBlock->info.numOfCols == 2) { + } else if (numOfCols == 2) { + } } @@ -1120,34 +1124,36 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF void blockDataCleanup(SSDataBlock* pDataBlock) { pDataBlock->info.rows = 0; - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); colInfoDataCleanup(p, pDataBlock->info.capacity); } } -int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows) { - ASSERT(numOfRows); - if (0 == numOfRows || numOfRows <= existRows) { +static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo *pBlockInfo, uint32_t numOfRows) { + ASSERT(numOfRows > 0 && pBlockInfo->capacity >= pBlockInfo->rows); + if (numOfRows < pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } + int32_t existedRows = pBlockInfo->rows; + if (IS_VAR_DATA_TYPE(pColumn->info.type)) { char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); - if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } pColumn->varmeta.offset = (int32_t*)tmp; - memset(&pColumn->varmeta.offset[existRows], 0, sizeof(int32_t) * (numOfRows - existRows)); + memset(&pColumn->varmeta.offset[existedRows], 0, sizeof(int32_t) * (numOfRows - existedRows)); } else { char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - int32_t oldLen = BitmapLen(existRows); + int32_t oldLen = BitmapLen(existedRows); pColumn->nullbitmap = tmp; memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); @@ -1160,8 +1166,8 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, ui if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - memset(tmp + pColumn->info.bytes * existRows, 0, pColumn->info.bytes * (numOfRows - existRows)); - + + memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); pColumn->pData = tmp; } @@ -1178,6 +1184,11 @@ void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { } } +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { + SDataBlockInfo info = {0}; + return doEnsureCapacity(pColumn, &info, numOfRows); +} + int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; ASSERT(numOfRows > 0); @@ -1188,9 +1199,10 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { pDataBlock->info.capacity = numOfRows; - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - code = colInfoDataEnsureCapacity(p, pDataBlock->info.rows, numOfRows); + code = doEnsureCapacity(p, &pDataBlock->info, numOfRows); if (code) { return code; } @@ -1214,40 +1226,34 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { return NULL; } - int32_t numOfCols = pDataBlock->info.numOfCols; - - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - - pBlock->info.numOfCols = numOfCols; - pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; - pBlock->info.rowSize = pDataBlock->info.rowSize; - pBlock->info.groupId = pDataBlock->info.groupId; - pBlock->info.childId = pDataBlock->info.childId; - pBlock->info.type = pDataBlock->info.type; + SSDataBlock* pBlock = createDataBlock(); + pBlock->info = pDataBlock->info; + pBlock->info.rows = 0; + pBlock->info.capacity = 0; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colInfo = {0}; SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - colInfo.info = p->info; - colInfo.hasNull = true; - taosArrayPush(pBlock->pDataBlock, &colInfo); + SColumnInfoData colInfo = {.hasNull = true, .info = p->info}; + blockDataAppendColInfo(pBlock, &colInfo); } if (copyData) { + int32_t code = blockDataEnsureCapacity(pBlock, pDataBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + blockDataDestroy(pBlock); + return NULL; + } + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); - - int32_t code = colInfoDataEnsureCapacity(pDst, 0, pDataBlock->info.rows); - if (code != TSDB_CODE_SUCCESS) { - return NULL; - } - if (pSrc->pData == NULL) { continue; } - colDataAssign(pDst, pSrc, pDataBlock->info.rows); + + colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); } pBlock->info.rows = pDataBlock->info.rows; @@ -1257,16 +1263,74 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { return pBlock; } +SSDataBlock* createDataBlock() { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + if (pBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + if (pBlock->pDataBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pBlock); + } + + return pBlock; +} + +int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData) { + ASSERT(pBlock != NULL && pColInfoData != NULL); + if (pBlock->pDataBlock == NULL) { + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + if (pBlock->pDataBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + } + + void* p = taosArrayPush(pBlock->pDataBlock, pColInfoData); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + ASSERT(pColInfoData->info.type != 0); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + pBlock->info.hasVarCol = true; + } + + pBlock->info.rowSize += pColInfoData->info.bytes; + return TSDB_CODE_SUCCESS; +} + +SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) { + SColumnInfoData col = {.hasNull = true}; + col.info.colId = colId; + col.info.type = type; + col.info.bytes = bytes; + + return col; +} + +SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index) { + ASSERT(pBlock != NULL); + if (index >= taosArrayGetSize(pBlock->pDataBlock)) { + return NULL; + } + + return taosArrayGet(pBlock->pDataBlock, index); +} + size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { - int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock->info.numOfCols); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols); int32_t rowSize = pBlock->info.rowSize; - int32_t nRows = payloadSize / rowSize; // the true value must be less than the value of nRows int32_t additional = 0; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pCol->info.type)) { additional += nRows * sizeof(int32_t); @@ -1350,7 +1414,8 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { if (pBlock->info.rows <= n) { blockDataCleanup(pBlock); } else { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows); } @@ -1362,7 +1427,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int64_t tbUid = pBlock->info.uid; - int16_t numOfCols = pBlock->info.numOfCols; + int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int16_t hasVarCol = pBlock->info.hasVarCol; int32_t rows = pBlock->info.rows; int32_t sz = taosArrayGetSize(pBlock->pDataBlock); @@ -1397,8 +1462,10 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { int32_t sz; - buf = taosDecodeFixedI64(buf, &pBlock->info.uid); - buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols); + int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + + buf = taosDecodeFixedU64(buf, &pBlock->info.uid); + buf = taosDecodeFixedI16(buf, &numOfCols); buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol); buf = taosDecodeFixedI32(buf, &pBlock->info.rows); buf = taosDecodeFixedI32(buf, &sz); @@ -1507,12 +1574,13 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); for (int32_t j = 0; j < rows; j++) { printf("%s |", flag); - for (int32_t k = 0; k < colNum; k++) { + for (int32_t k = 0; k < numOfCols; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); if (colDataIsNull(pColInfoData, rows, j, NULL)) { @@ -1565,7 +1633,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks int32_t bufSize = sizeof(SSubmitReq); for (int32_t i = 0; i < sz; ++i) { SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info; - bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols)); + + int32_t numOfCols = taosArrayGetSize(pDataBlocks); + bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(numOfCols)); bufSize += sizeof(SSubmitBlk); } @@ -1583,10 +1653,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks for (int32_t i = 0; i < sz; ++i) { SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; - int32_t rowSize = pDataBlock->info.rowSize; - int64_t groupId = pDataBlock->info.groupId; +// int32_t rowSize = pDataBlock->info.rowSize; +// int64_t groupId = pDataBlock->info.groupId; if (colNum <= 1) { // invalid if only with TS col @@ -1819,9 +1889,6 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t taosArraySetSize(pBlock->pDataBlock, numOfCols); } - pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock); - ASSERT(pBlock->pDataBlock->size >= numOfCols); - for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData->info.type = *(int16_t*)pStart; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 2adb558d01..8460d71e56 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -88,26 +88,15 @@ TEST(testCase, toInteger_test) { } TEST(testCase, Datablock_test) { - SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - b->info.numOfCols = 2; - b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SSDataBlock* b = createDataBlock(); - SColumnInfoData infoData = {0}; - infoData.info.bytes = 4; - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.colId = 1; - - infoData.pData = (char*)taosMemoryCalloc(40, infoData.info.bytes); - infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (40 / 8)); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1); taosArrayPush(b->pDataBlock, &infoData); + blockDataAppendColInfo(b, &infoData); - SColumnInfoData infoData1 = {0}; - infoData1.info.bytes = 40; - infoData1.info.type = TSDB_DATA_TYPE_BINARY; - infoData1.info.colId = 2; - - infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(40, sizeof(uint32_t)); - taosArrayPush(b->pDataBlock, &infoData1); + SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2); + blockDataAppendColInfo(b, &infoData1); + blockDataEnsureCapacity(b, 40); char* str = "the value of: %d"; char buf[128] = {0}; @@ -222,28 +211,17 @@ TEST(testCase, non_var_dataBlock_split_test) { #endif TEST(testCase, var_dataBlock_split_test) { - SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - b->info.numOfCols = 2; - b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - int32_t numOfRows = 1000000; - SColumnInfoData infoData = {0}; - infoData.info.bytes = 4; - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.colId = 1; + SSDataBlock* b = createDataBlock(); - infoData.pData = (char*)taosMemoryCalloc(numOfRows, infoData.info.bytes); - infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (numOfRows / 8)); - taosArrayPush(b->pDataBlock, &infoData); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1); + blockDataAppendColInfo(b, &infoData); - SColumnInfoData infoData1 = {0}; - infoData1.info.bytes = 40; - infoData1.info.type = TSDB_DATA_TYPE_BINARY; - infoData1.info.colId = 2; + SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2); + blockDataAppendColInfo(b, &infoData1); - infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(numOfRows, sizeof(uint32_t)); - taosArrayPush(b->pDataBlock, &infoData1); + blockDataEnsureCapacity(b, numOfRows); char buf[41] = {0}; char buf1[100] = {0}; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index fed6a5a721..27de3883e9 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -236,7 +236,6 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {0}; @@ -271,7 +270,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + - blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols); + blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 70d89102e6..b58450ad44 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -149,8 +149,8 @@ int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidL int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, - int32_t *pNumOfRows, int16_t *pNumOfCols); +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, + int32_t *pNumOfRows); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 7c75d88a83..e1c3322a04 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs // TODO enable compress int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); @@ -74,7 +74,6 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR if (pDataBlock == NULL) break; ASSERT(pDataBlock->info.rows != 0); - ASSERT(pDataBlock->info.numOfCols != 0); tqAddBlockDataToRsp(pDataBlock, pRsp); if (pRsp->withTbName) { @@ -88,8 +87,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, - &block.info.numOfCols) < 0) { + if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } @@ -106,8 +104,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, - &block.info.numOfCols) < 0) { + if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 0c38d6442b..fa0eb1d458 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -123,8 +123,8 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) { return false; } -int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, - int32_t* pNumOfRows, int16_t* pNumOfCols) { +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, + int32_t* pNumOfRows) { *pUid = 0; // TODO set to real sversion @@ -161,23 +161,14 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); if (colNumNeed == 0) { - *ppCols = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData)); - if (*ppCols == NULL) { - return -1; - } - int32_t colMeta = 0; while (colMeta < pSchemaWrapper->nCols) { SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - SColumnInfoData colInfo = {0}; - colInfo.info.bytes = pColSchema->bytes; - colInfo.info.colId = pColSchema->colId; - colInfo.info.type = pColSchema->type; - - if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { goto FAIL; } - taosArrayPush(*ppCols, &colInfo); colMeta++; } } else { @@ -185,11 +176,6 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p colNumNeed = pSchemaWrapper->nCols; } - *ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); - if (*ppCols == NULL) { - return -1; - } - int32_t colMeta = 0; int32_t colNeed = 0; while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { @@ -201,23 +187,22 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p } else if (colIdSchema > colIdNeed) { colNeed++; } else { - SColumnInfoData colInfo = {0}; - colInfo.info.bytes = pColSchema->bytes; - colInfo.info.colId = pColSchema->colId; - colInfo.info.type = pColSchema->type; - - if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { goto FAIL; } - taosArrayPush(*ppCols, &colInfo); colMeta++; colNeed++; } } } - int32_t colActual = taosArrayGetSize(*ppCols); - *pNumOfCols = colActual; + if (blockDataEnsureCapacity(pBlock, *pNumOfRows) < 0) { + goto FAIL; + } + + int32_t colActual = blockDataGetNumOfCols(pBlock); // TODO in stream shuffle case, fetch groupId *pGroupId = 0; @@ -234,7 +219,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p tdSTSRowIterReset(&iter, row); // get all wanted col of that block for (int32_t i = 0; i < colActual; i++) { - SColumnInfoData* pColData = taosArrayGet(*ppCols, i); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SCellVal sVal = {0}; if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; @@ -246,8 +231,9 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p curRow++; } return 0; -FAIL: - if (*ppCols) taosArrayDestroy(*ppCols); + +FAIL: // todo refactor here +// if (*ppCols) taosArrayDestroy(*ppCols); return -1; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b628f0dde5..393cb9f749 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -44,7 +44,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.ctb.suid = suid; STagVal tagVal = { - .cid = pDataBlock->info.numOfCols + 1, + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = (int64_t)pDataBlock->info.groupId, }; @@ -109,7 +109,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.ctb.suid = suid; STagVal tagVal = { - .cid = pDataBlock->info.numOfCols + 1, + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = (int64_t)pDataBlock->info.groupId, }; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ce73246e51..1a623f1978 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -16,13 +16,11 @@ #include "tsdb.h" #include "vnode.h" -#define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) -#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) +#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pResBlock->pDataBlock))) #define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ ((SDataBlockInfo){.window = {.skey = (_block)->minKey.ts, .ekey = (_block)->maxKey.ts}, \ - .numOfCols = (_block)->numOfCols, \ .rows = (_block)->numOfRows, \ .uid = (_checkInfo)->tableId}) @@ -115,7 +113,8 @@ typedef struct STsdbReadHandle { // SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time // SColumnDataAgg** pstatis;// the ptr array list to return to caller int32_t numOfBlocks; - SArray* pColumns; // column list, SColumnInfoData array list + SSDataBlock* pResBlock; +// SArray* pColumns; // column list, SColumnInfoData array list bool locateStart; int32_t outputCapacity; int32_t realNumOfRows; @@ -177,7 +176,7 @@ static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) { SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); taosArrayPush(pIdList, &pCol->info.colId); } @@ -425,23 +424,21 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* } // todo: use list instead of array? - pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); - if (pReadHandle->pColumns == NULL) { + pReadHandle->pResBlock = createDataBlock(); + if (pReadHandle->pResBlock == NULL) { goto _end; } for (int32_t i = 0; i < pCond->numOfCols; ++i) { - SColumnInfoData colInfo = {{0}, 0}; - colInfo.info = pCond->colList[i]; - - int32_t code = colInfoDataEnsureCapacity(&colInfo, 0, pReadHandle->outputCapacity); - if (code != TSDB_CODE_SUCCESS) { + SColumnInfoData colInfo = {.info = pCond->colList[i], 0}; + int32_t code = blockDataAppendColInfo(pReadHandle->pResBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS){ goto _end; } - - taosArrayPush(pReadHandle->pColumns, &colInfo); } + blockDataEnsureCapacity(pReadHandle->pResBlock, pReadHandle->outputCapacity); + pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); size_t size = taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn); @@ -1536,12 +1533,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t int32_t trueEnd = ascScan ? end : start; int32_t step = ascScan ? 1 : -1; - int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); // data in buffer has greater timestamp, copy data in file block int32_t i = 0, j = 0; while (i < requiredNumOfCols && j < pCols->numOfCols) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); SDataCol* src = &pCols->cols[j]; if (src->colId < pColInfo->info.colId) { @@ -1592,7 +1589,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t } while (i < requiredNumOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colDataAppendNNULL(pColInfo, numOfRows, num); i++; } @@ -1674,7 +1671,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa int32_t i = 0, j = 0, k = 0; while (i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); int32_t colIdOfRow1; if (j >= numOfColsOfRow1) { @@ -1805,7 +1802,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (mergeOption == 1) { while (i < numOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colDataAppend(pColInfo, *curRow, NULL, true); ++i; } @@ -1861,7 +1858,7 @@ static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) { assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey); } - SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0); assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows - 1]); } else { @@ -2771,7 +2768,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; int curRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); win->skey = TSKEY_INITIAL_VAL; @@ -2901,7 +2898,7 @@ static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) { if (exists) { tsdbRetrieveDataBlock((tsdbReaderT*)pTsdbReadHandle, NULL); if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0); assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); } @@ -3003,7 +3000,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // // int32_t i = 0, j = 0; // while(i < tgNumOfCols && j < numOfCols) { -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // if (pTable->lastCols[j].colId < pColInfo->info.colId) { // j++; // continue; @@ -3066,7 +3063,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // continue; // } // -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // // if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -3091,7 +3088,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // // // leave the real ts column as the last row, because last function only (not stable) use the last row as res // if (priKey != TSKEY_INITIAL_VAL) { -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, priIdx); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; // // *(TSKEY *)pData = priKey; @@ -3101,7 +3098,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // continue; // } // -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // // assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID); @@ -3161,9 +3158,9 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { bool tsdbNextDataBlock(tsdbReaderT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)pHandle; - size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pColumns); + size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colInfoDataCleanup(pColInfo, pTsdbReadHandle->outputCapacity); } @@ -3250,7 +3247,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next; // // for (int32_t i = 0; i < numOfCols; ++i) { -// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); +// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // // SColumnInfoData colInfo = {{0}, 0}; // colInfo.info = pCol->info; @@ -3280,7 +3277,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // } // // for (int32_t i = 0; i < cond.numOfCols; ++i) { -// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); +// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); // } // @@ -3342,7 +3339,7 @@ bool tsdbGetExternalRow(tsdbReaderT pHandle) { int32_t numOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i); memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); @@ -3617,13 +3614,13 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { */ STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; if (pHandle->cur.fid == INT32_MIN) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; if (pHandle->cur.mixBlock) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); assert(pHandle->realNumOfRows <= binfo.rows); @@ -3633,7 +3630,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && pBlockLoadInfo->uid == pCheckInfo->tableId) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { // only load the file block SBlock* pBlock = pBlockInfo->compBlock; if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) { @@ -3641,7 +3638,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { } int32_t numOfRows = doCopyRowsFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1); - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } } } @@ -3692,7 +3689,7 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { return; } - pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); + pTsdbReadHandle->pResBlock->pDataBlock = doFreeColumnInfoData(pTsdbReadHandle->pResBlock->pDataBlock); taosArrayDestroy(pTsdbReadHandle->suppInfo.defaultLoadColumn); taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo); diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 2c1e6fbbbd..2a8c7a583c 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -435,7 +435,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { SSDataBlock *pDataBlock = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); EXPECT_NE(pDataBlock, nullptr); pDataBlock->pBlockAgg = NULL; - pDataBlock->info.numOfCols = tSmaNumOfCols; + taosArrayGetSize(pDataBlock->pDataBlock) = tSmaNumOfCols; pDataBlock->info.rows = tSmaNumOfRows; pDataBlock->info.groupId = tSmaGroupId + g; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index c6a0e52a42..66dc2bcaaf 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -30,31 +30,19 @@ static int32_t getSchemaBytes(const SSchema* pSchema) { } static SSDataBlock* buildDescResultDataBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = DESCRIBE_RESULT_COLS; - pBlock->info.hasVarCol = true; + SSDataBlock* pBlock = createDataBlock(); - pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_FIELD_LEN, 1); + blockDataAppendColInfo(pBlock, &infoData); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_FIELD_LEN; + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_TYPE_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; - - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_NOTE_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, tDataTypes[TSDB_DATA_TYPE_INT].bytes, 3); + blockDataAppendColInfo(pBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_NOTE_LEN, 4); + blockDataAppendColInfo(pBlock, &infoData); return pBlock; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index ae0f669d65..091f300428 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1190,17 +1190,12 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE; - - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - taosArrayPush(pBlock->pDataBlock, &infoData); + SSDataBlock *pBlock = createDataBlock(); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1); + blockDataAppendColInfo(pBlock, &infoData); + blockDataEnsureCapacity(pBlock, rowNum); SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); - pInfoData->hasNull = false; - colInfoDataEnsureCapacity(pInfoData, 0, rowNum); char buf[1024] = {0}; for (int32_t i = 0; i < rowNum; ++i) { @@ -1210,9 +1205,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { colDataAppend(pInfoData, i, buf, false); } - pBlock->info.numOfCols = 1; pBlock->info.rows = rowNum; - pBlock->info.hasVarCol = true; int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); @@ -1226,7 +1219,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htonl(rowNum); int32_t len = 0; - blockCompressEncode(pBlock, rsp->data, &len, pBlock->info.numOfCols, 0); + blockCompressEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 33b7811e6c..5ba275c7b2 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -75,7 +75,7 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = 0; pEntry->numOfRows = pInput->pData->info.rows; - pEntry->numOfCols = pInput->pData->info.numOfCols; + pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock); pEntry->dataLen = sizeof(SDeleterRes); ASSERT(1 == pEntry->numOfRows); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 566abb1451..60560e49dd 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -183,34 +183,24 @@ SArray* createSortInfo(SNodeList* pNodeList) { SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { int32_t numOfCols = LIST_LENGTH(pNode->pSlots); - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + SSDataBlock* pBlock = createDataBlock(); pBlock->info.blockId = pNode->dataBlockId; - pBlock->info.rowSize = pNode->totalRowSize; // todo ?? pBlock->info.type = STREAM_INVALID; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = {{0}}; SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); // if (!pDescNode->output) { // todo disable it temporarily // continue; // } - idata.info.type = pDescNode->dataType.type; - idata.info.bytes = pDescNode->dataType.bytes; + SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); idata.info.scale = pDescNode->dataType.scale; - idata.info.slotId = pDescNode->slotId; idata.info.precision = pDescNode->dataType.precision; - if (IS_VAR_DATA_TYPE(idata.info.type)) { - pBlock->info.hasVarCol = true; - } - - taosArrayPush(pBlock->pDataBlock, &idata); + blockDataAppendColInfo(pBlock, &idata); } - pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock); return pBlock; } @@ -600,7 +590,7 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray if (p->info.colId == pmInfo->colId) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, p, pBlock->info.rows); + colDataAssign(pDst, p, pBlock->info.rows, &pBlock->info); i++; j++; } else if (p->info.colId < pmInfo->colId) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index cf72dfd796..e805de3727 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -68,6 +68,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayClear(p->pDataBlock); taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock); + printf("size------------->%ld, %ld, %p, \n", taosArrayGetSize(pDataBlock->pDataBlock), taosArrayGetSize(pDataBlock->pDataBlock) * pDataBlock->pDataBlock->elemSize, + pDataBlock->pDataBlock); taosArrayPush(pInfo->pBlockLists, &p); } } else { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 20dac85223..72c480a81b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -356,7 +356,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; pColData->info.bytes = sizeof(int64_t); - colInfoDataEnsureCapacity(pColData, 0, 5); + colInfoDataEnsureCapacity(pColData, 5); colDataAppendInt64(pColData, 0, &pQueryWindow->skey); colDataAppendInt64(pColData, 1, &pQueryWindow->ekey); @@ -458,7 +458,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc pColInfo = pInput->pData[paramIndex]; } - colInfoDataEnsureCapacity(pColInfo, 0, numOfRows); + colInfoDataEnsureCapacity(pColInfo, numOfRows); int8_t type = pFuncParam->param.nType; if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { @@ -579,7 +579,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0], pfCtx->input.numOfRows); } else { - colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); + colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows, &pResult->info); } numOfRows = pfCtx->input.numOfRows; @@ -1186,7 +1186,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc // } // if (pQueryAttr->pFilters != NULL) { -// filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); +// filterSetColFieldData(pQueryAttr->pFilters, taosArrayGetSize(pBlock->pDataBlock), pBlock->pDataBlock); // } // if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { @@ -1321,7 +1321,8 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { // todo move to the initialization function int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); - SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock}; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + SFilterColumnParam param1 = {.numOfCols = numOfCols, .pDataBlock = pBlock->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); int8_t* rowRes = NULL; @@ -1343,7 +1344,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR int32_t totalRows = pBlock->info.rows; SSDataBlock* px = createOneDataBlock(pBlock, true); - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); // it is a reserved column for scalar function, and no data in this column yet. @@ -2064,29 +2066,20 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo pStart += sizeof(SSysTableSchema); } - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; - + SSDataBlock* pBlock = createDataBlock(); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = {0}; - - idata.info.type = pSchema[i].type; - idata.info.bytes = pSchema[i].bytes; - idata.info.colId = pSchema[i].colId; - idata.hasNull = true; - - taosArrayPush(pBlock->pDataBlock, &idata); - if (IS_VAR_DATA_TYPE(idata.info.type)) { - pBlock->info.hasVarCol = true; - } + SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); + blockDataAppendColInfo(pBlock, &idata); } blockCompressDecode(pBlock, numOfCols, numOfRows, pStart); + blockDataEnsureCapacity(pRes, numOfRows); + + relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); // data from mnode pRes->info.rows = numOfRows; - relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); + taosArrayDestroy(pBlock->pDataBlock); taosMemoryFree(pBlock); // blockDataDestroy(pBlock); @@ -2412,15 +2405,15 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode tsem_init(&pInfo->ready, 0, 0); - pInfo->seqLoadData = false; + pInfo->seqLoadData = false; pInfo->pTransporter = pTransporter; - pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); - pOperator->name = "ExchangeOperator"; + pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pInfo->pResult->info.numOfCols; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pResult->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, @@ -2542,9 +2535,6 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock SSortedMergeOperatorInfo* pInfo = pOperator->info; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - // pCtx[i].size = 1; - } for (int32_t i = 0; i < pBlock->info.rows; ++i) { if (!pInfo->hasGroupVal) { @@ -2653,7 +2643,7 @@ SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 064a36df1c..4e4aaba7f4 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -559,8 +559,8 @@ uint64_t calcGroupId(char* pData, int32_t len) { } int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { - size_t numOfCols = pBlock->info.numOfCols; - int32_t* offset = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(int32_t)); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t)); offset[0] = sizeof(int32_t) + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index eed357c86d..8c6797759e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -203,7 +203,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pCost->skipBlocks += 1; // clear all data in pBlock that are set when handing the previous block - for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); pcol->pData = NULL; } @@ -217,7 +217,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); if (allColumnsHaveAgg == true) { - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); // todo create this buffer during creating operator if (pBlock->pBlockAgg == NULL) { @@ -316,7 +316,7 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_ SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); - colInfoDataEnsureCapacity(pColInfoData, 0, pBlock->info.rows); + colInfoDataEnsureCapacity(pColInfoData, pBlock->info.rows); colInfoDataCleanup(pColInfoData, pBlock->info.rows); int32_t functionId = pExpr->pExpr->_function.functionId; @@ -354,10 +354,8 @@ void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* p struct SScalarFuncExecFuncs fpSet = {0}; fmGetScalarFuncExecFuncs(functionId, &fpSet); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_BIGINT; - infoData.info.bytes = sizeof(uint64_t); - colInfoDataEnsureCapacity(&infoData, 0, 1); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_BIGINT, sizeof(uint64_t), 1); + colInfoDataEnsureCapacity(&infoData, 1); colDataAppendInt64(&infoData, 0, (int64_t*)&pBlock->info.uid); SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData}; @@ -785,7 +783,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { } static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) { - for (int32_t j = 0; j < source->info.numOfCols; j++) { + for (int32_t j = 0; j < taosArrayGetSize(source->pDataBlock); j++) { SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j); SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j); if (colDataIsNull_s(pSourceCol, sourceRowId)) { @@ -857,7 +855,6 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); blockDataEnsureCapacity(pUpdateBlock, size); - ASSERT(pBlock->info.numOfCols == pUpdateBlock->info.numOfCols); int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex); pInfo->groupId = getGroupId(pInfo->pOperatorDumy, pBlock, rowId); @@ -968,19 +965,20 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); while (tqNextDataBlock(pInfo->streamBlockReader)) { - SArray* pCols = NULL; + SSDataBlock block = {0}; uint64_t groupId = 0; uint64_t uid = 0; int32_t numOfRows = 0; - int16_t outputCol = 0; - int32_t code = tqRetrieveDataBlock(&pCols, pInfo->streamBlockReader, &groupId, &uid, &numOfRows, &outputCol); + int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows); if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { pTaskInfo->code = code; return NULL; } + blockDataEnsureCapacity(pInfo->pRes, numOfRows); + pInfo->pRes->info.groupId = groupId; pInfo->pRes->info.rows = numOfRows; pInfo->pRes->info.uid = uid; @@ -999,6 +997,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->pRes->info.groupId = *groupIdPre; } + // todo extract method for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); if (!pColMatchInfo->output) { @@ -1006,8 +1005,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } bool colExists = false; - for (int32_t j = 0; j < taosArrayGetSize(pCols); ++j) { - SColumnInfoData* pResCol = taosArrayGet(pCols, j); + for (int32_t j = 0; j < blockDataGetNumOfCols(&block); ++j) { + SColumnInfoData* pResCol = bdGetColumnInfoData(&block, j); if (pResCol->info.colId == pColMatchInfo->colId) { taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); colExists = true; @@ -1018,7 +1017,6 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { // the required column does not exists in submit block, let's set it to be all null value if (!colExists) { SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); - colInfoDataEnsureCapacity(pDst, 0, pBlockInfo->rows); colDataAppendNNULL(pDst, 0, pBlockInfo->rows); } } @@ -1164,7 +1162,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pInfo->pRes->info.numOfCols; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = @@ -1292,7 +1290,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i); if (keep) { - colDataAssign(pDest, pSrc, pInfo->pRes->info.rows); + colDataAssign(pDest, pSrc, pInfo->pRes->info.rows, &px->info); numOfRow = pInfo->pRes->info.rows; } else if (NULL != rowRes) { numOfRow = 0; @@ -1322,8 +1320,6 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { } static SSDataBlock* buildSysTableMetaBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - size_t size = 0; const SSysTableMeta* pMeta = NULL; getInfosDbMeta(&pMeta, &size); @@ -1336,19 +1332,12 @@ static SSDataBlock* buildSysTableMetaBlock() { } } - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - + SSDataBlock* pBlock = createDataBlock(); for (int32_t i = 0; i < pMeta[index].colNum; ++i) { - SColumnInfoData colInfoData = {0}; - colInfoData.info.colId = i + 1; - colInfoData.info.type = pMeta[index].schema[i].type; - colInfoData.info.bytes = pMeta[index].schema[i].bytes; - taosArrayPush(pBlock->pDataBlock, &colInfoData); + SColumnInfoData colInfoData = createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); + blockDataAppendColInfo(pBlock, &colInfoData); } - pBlock->info.numOfCols = pMeta[index].colNum; - pBlock->info.hasVarCol = true; - return pBlock; } @@ -1696,7 +1685,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pResBlock->info.numOfCols; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = @@ -2033,7 +2022,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc pCost->skipBlocks += 1; // clear all data in pBlock that are set when handing the previous block - for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); pcol->pData = NULL; } @@ -2048,7 +2037,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc tsdbRetrieveDataBlockStatisInfo(reader, &pColAgg, &allColumnsHaveAgg); if (allColumnsHaveAgg == true) { - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); // todo create this buffer during creating operator if (pBlock->pBlockAgg == NULL) { diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 9d13276e6d..feb81b1708 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -78,7 +78,7 @@ _error: } void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); bool isNull = tsortIsNullVal(pTupleHandle, i); if (isNull) { @@ -118,6 +118,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i } if (p->info.rows > 0) { + // todo extract function to handle this int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); @@ -125,7 +126,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; @@ -317,7 +318,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 110c2d6a8f..1d82d6a644 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -50,7 +50,7 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) { // the first are always the timestamp column, so start from the second column. - for (int32_t i = 1; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i); colDataAppendNULL(p, rowIndex); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index bde4aa18e8..2dbdbd7069 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1821,7 +1821,7 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB return TSDB_CODE_OUT_OF_MEMORY; } - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); @@ -2234,9 +2234,6 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) } static void clearUpdateDataBlock(SSDataBlock* pBlock) { - if (pBlock->info.rows <= 0) { - return; - } blockDataCleanup(pBlock); } @@ -2245,12 +2242,14 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol clearUpdateDataBlock(pDest); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex); + // copy timestamp column - colDataAssign(pDestCol, pSourceCol, pSource->info.rows); - for (int32_t i = 1; i < pDest->info.numOfCols; i++) { + colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info); + for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) { SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i); colDataAppendNNULL(pCol, 0, pSource->info.rows); } + pDest->info.rows = pSource->info.rows; pDest->info.groupId = pSource->info.groupId; pDest->info.type = pSource->info.type; @@ -2919,7 +2918,7 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false); - for (int32_t i = 1; i < pBlock->info.numOfCols; i++) { + for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); i++) { pColInfoData = taosArrayGet(pBlock->pDataBlock, i); colDataAppendNULL(pColInfoData, pBlock->info.rows); } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index f21cad2dd6..caf567884f 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -146,7 +146,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock); // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation - int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock->info.numOfCols))/rowSize; + int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)))/rowSize; ASSERT(numOfRows > 0); return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); } @@ -177,7 +177,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { return terrno; } - int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t); + int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t); assert(size <= getBufPageSize(pHandle->pBuf)); blockDataToBuf(pPage, p); @@ -251,7 +251,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int } static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i); @@ -498,7 +498,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { return terrno; } - int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t); + int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t); assert(size <= getBufPageSize(pHandle->pBuf)); blockDataToBuf(pPage, pDataBlock); diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index dba2fb897a..3c12889aca 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -62,18 +62,11 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = createDataBlock(); - pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); - - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pInfo->pBlock, &colInfo); + blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage); // SColumnInfoData colInfo1 = {0}; // colInfo1.info.type = TSDB_DATA_TYPE_BINARY; @@ -115,7 +108,6 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { } pBlock->info.rows = pInfo->numOfRowsPerPage; - pBlock->info.numOfCols = 1; pInfo->current += 1; return pBlock; @@ -128,28 +120,15 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = createDataBlock(); - pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1); + blockDataAppendColInfo(pInfo->pBlock, &colInfo); - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP; - colInfo.info.bytes = sizeof(int64_t); - colInfo.info.colId = 1; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); - // colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + SColumnInfoData colInfo1 = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2); + blockDataAppendColInfo(pInfo->pBlock, &colInfo1); - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); - - SColumnInfoData colInfo1 = {0}; - colInfo1.info.type = TSDB_DATA_TYPE_INT; - colInfo1.info.bytes = 4; - colInfo1.info.colId = 2; - - colInfo1.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo1.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); - - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); + blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage); } else { blockDataCleanup(pInfo->pBlock); } @@ -185,7 +164,6 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { } pBlock->info.rows = pInfo->numOfRowsPerPage; - pBlock->info.numOfCols = 1; pInfo->current += 1; diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index 66ed078bbe..70def78263 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -62,25 +62,21 @@ SSDataBlock* getSingleColDummyBlock(void* param) { return NULL; } - SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SSDataBlock* pBlock = createDataBlock(); SColumnInfoData colInfo = {0}; colInfo.info.type = pInfo->type; if (pInfo->type == TSDB_DATA_TYPE_NCHAR){ colInfo.info.bytes = TSDB_NCHAR_SIZE * VARCOUNT + VARSTR_HEADER_SIZE; - colInfo.varmeta.offset = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); } else if(pInfo->type == TSDB_DATA_TYPE_BINARY) { colInfo.info.bytes = VARCOUNT + VARSTR_HEADER_SIZE; - colInfo.varmeta.offset = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); } else{ colInfo.info.bytes = tDataTypes[pInfo->type].bytes; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->pageRows, colInfo.info.bytes)); - colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8)); } colInfo.info.colId = 1; - taosArrayPush(pBlock->pDataBlock, &colInfo); + blockDataAppendColInfo(pBlock, &colInfo); + blockDataEnsureCapacity(pBlock, pInfo->pageRows); for (int32_t i = 0; i < pInfo->pageRows; ++i) { SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); @@ -128,7 +124,6 @@ SSDataBlock* getSingleColDummyBlock(void* param) { } pBlock->info.rows = pInfo->pageRows; - pBlock->info.numOfCols = 1; return pBlock; } @@ -354,15 +349,10 @@ TEST(testCase, ordered_merge_sort_Test) { SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); taosArrayPush(orderInfo, &oi); - SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - taosArrayPush(pBlock->pDataBlock, &colInfo); + SSDataBlock* pBlock = createDataBlock(); + for (int32_t i = 0; i < 1; ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); } SSortHandle* phandle = tsortCreateSortHandle(orderInfo, SORT_MULTISOURCE_MERGE, 1024, 5, pBlock,"test_abc"); diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index c3d61d426d..669c2635b5 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -28,19 +28,6 @@ extern "C" { #include "function.h" #include "tudf.h" - -extern SAggFunctionInfo aggFunc[35]; - -#define FUNCSTATE_SO 0x0u -#define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM -#define FUNCSTATE_STREAM 0x2u // function avail for stream -#define FUNCSTATE_STABLE 0x4u // function avail for super table -#define FUNCSTATE_NEED_TS 0x8u // timestamp is required during query processing -#define FUNCSTATE_SELECTIVITY 0x10u // selectivity functions, can exists along with tag columns - -#define BASIC_FUNC_SO FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE -#define BASIC_FUNC_MO FUNCSTATE_MO | FUNCSTATE_STREAM | FUNCSTATE_STABLE - #define AVG_FUNCTION_INTER_BUFFER_SIZE 50 #define DATA_SET_FLAG ',' // to denote the output area has data, not null value diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 076ae61460..784017fb05 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1427,11 +1427,13 @@ void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) { int32_t pageId = pTuplePos->pageId; int32_t offset = pTuplePos->offset; + + int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock); if (pTuplePos->pageId != -1) { SFilePage* pPage = getBufPage(pCtx->pBuf, pageId); bool* nullList = (bool*)((char*)pPage + offset); - char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool)); + char* pStart = (char*)(nullList + numOfCols * sizeof(bool)); // todo set the offset value to optimize the performance. for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { @@ -3018,7 +3020,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = NULL; - int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool); + int32_t completeRowSize = pSrcBlock->info.rowSize + (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock) * sizeof(bool); if (pCtx->curBufPage == -1) { pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage); @@ -3036,8 +3038,8 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS // keep the current row data, extract method int32_t offset = 0; bool* nullList = (bool*)((char*)pPage + pPage->num); - char* pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols); - for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + char* pStart = (char*)(nullList + sizeof(bool) * (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock)); + for (int32_t i = 0; i < (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock); ++i) { SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); bool isNull = colDataIsNull_s(pCol, rowIndex); if (isNull) { @@ -3066,11 +3068,13 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId); + int32_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); + bool* nullList = (bool*)((char*)pPage + pPos->offset); - char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool)); + char* pStart = (char*)(nullList + numOfCols * sizeof(bool)); int32_t offset = 0; - for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) { offset += pCol->info.bytes; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index f6ae027e48..0ecad278e3 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -763,8 +763,7 @@ void freeUdfInterBuf(SUdfInterBuf *buf) { int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) { udfBlock->numOfRows = block->info.rows; - udfBlock->numOfCols = block->info.numOfCols; - udfBlock->udfCols = taosMemoryCalloc(udfBlock->numOfCols, sizeof(SUdfColumn*)); + udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn*)); for (int32_t i = 0; i < udfBlock->numOfCols; ++i) { udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn)); SColumnInfoData *col= (SColumnInfoData*)taosArrayGet(block->pDataBlock, i); @@ -799,7 +798,6 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo } int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { - block->info.numOfCols = 1; block->info.rows = udfCol->colData.numOfRows; block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type); @@ -830,25 +828,19 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) { output->info.rows = input->numOfRows; - output->info.numOfCols = numOfCols; - bool hasVarCol = false; - for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) { - hasVarCol = true; - break; - } - } - output->info.hasVarCol = hasVarCol; - output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfCols; ++i) { taosArrayPush(output->pDataBlock, (input + i)->columnData); + + if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) { + output->info.hasVarCol = true; + } } return 0; } int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { - if (input->info.numOfCols != 1) { + if (taosArrayGetSize(input->pDataBlock) != 1) { fnError("scalar function only support one column"); return -1; } @@ -1090,23 +1082,14 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t numOfRows = pInput->numOfRows; - SSDataBlock tempBlock = {0}; - tempBlock.info.numOfCols = numOfCols; - tempBlock.info.rows = pInput->totalRows; - tempBlock.info.uid = pInput->uid; - bool hasVarCol = false; - tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - + SSDataBlock* pTempBlock = createDataBlock(); + pTempBlock->info.rows = pInput->totalRows; + pTempBlock->info.uid = pInput->uid; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData *col = pInput->pData[i]; - if (IS_VAR_DATA_TYPE(col->info.type)) { - hasVarCol = true; - } - taosArrayPush(tempBlock.pDataBlock, col); + blockDataAppendColInfo(pTempBlock, pInput->pData[i]); } - tempBlock.info.hasVarCol = hasVarCol; - SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows); + SSDataBlock *inputBlock = blockDataExtractBlock(pTempBlock, start, numOfRows); SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = session->bufSize, @@ -1131,7 +1114,9 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { } blockDataDestroy(inputBlock); - taosArrayDestroy(tempBlock.pDataBlock); + + taosArrayDestroy(pTempBlock->pDataBlock); + taosMemoryFree(pTempBlock); releaseUdfFuncHandle(pCtx->udfName); freeUdfInterBuf(&newState); diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index 15109db220..7395f1e14c 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -45,27 +45,27 @@ int scalarFuncTest() { for (int k = 0; k < 1; ++k) { SSDataBlock block = {0}; SSDataBlock *pBlock = █ - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; + for (int32_t i = 0; i < 1; ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); + } + + blockDataEnsureCapacity(pBlock, 1024); pBlock->info.rows = 1024; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows); - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(&colInfo, j, &j); - } - taosArrayPush(pBlock->pDataBlock, &colInfo); + + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, 0); + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(pCol, j, &j); } SScalarParam input = {0}; input.numOfRows = pBlock->info.rows; input.columnData = taosArrayGet(pBlock->pDataBlock, 0); + SScalarParam output = {0}; doCallUdfScalarFunc(handle, &input, 1, &output); taosArrayDestroy(pBlock->pDataBlock); + SColumnInfoData *col = output.columnData; for (int32_t i = 0; i < output.numOfRows; ++i) { if (i % 100 == 0) @@ -89,21 +89,18 @@ int aggregateFuncTest() { return -1; } - SSDataBlock block = {0}; - SSDataBlock *pBlock = █ - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; + SSDataBlock *pBlock = createDataBlock(); + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); + } + + blockDataEnsureCapacity(pBlock, 1024); pBlock->info.rows = 1024; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows); - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(&colInfo, j, &j); - } - taosArrayPush(pBlock->pDataBlock, &colInfo); + + SColumnInfoData* pColInfo = bdGetColumnInfoData(pBlock, 0); + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(pColInfo, j, &j); } SUdfInterBuf buf = {0}; @@ -121,6 +118,7 @@ int aggregateFuncTest() { freeUdfInterBuf(&resultBuf); doTeardownUdf(handle); + blockDataDestroy(pBlock); return 0; } diff --git a/source/libs/index/test/index_executor_tests.cpp b/source/libs/index/test/index_executor_tests.cpp index b88ffe5b8b..a0ac79a297 100644 --- a/source/libs/index/test/index_executor_tests.cpp +++ b/source/libs/index/test/index_executor_tests.cpp @@ -86,7 +86,7 @@ void sifAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *sl SColumnInfoData idata = {0}; idata.info = *colInfo; - colInfoDataEnsureCapacity(&idata, 0, rows); + colInfoDataEnsureCapacity(&idata, rows); taosArrayPush(res->pDataBlock, &idata); diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 4dd16098ff..c2d984c7dd 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -57,7 +57,7 @@ typedef struct SScalarCtx { #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); -SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); +SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); #define GET_PARAM_TYPE(_c) ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index a446aa7bdb..b8bcae161b 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3821,7 +3821,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData SScalarParam output = {0}; SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; - output.columnData = createColumnInfoData(&type, pSrc->info.rows); + output.columnData = sclCreateColumnInfoData(&type, pSrc->info.rows); SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 66e4af5ded..9890cff6f2 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -36,7 +36,7 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) { } -SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { +SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows) { SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); if (pColumnData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -48,7 +48,7 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { pColumnData->info.scale = pType->scale; pColumnData->info.precision = pType->precision; - int32_t code = colInfoDataEnsureCapacity(pColumnData, 0, numOfRows); + int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pColumnData); @@ -60,10 +60,10 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) { SScalarParam in = {.numOfRows = 1}; - in.columnData = createColumnInfoData(&pValueNode->node.resType, 1); + in.columnData = sclCreateColumnInfoData(&pValueNode->node.resType, 1); colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); - colInfoDataEnsureCapacity(out->columnData, 0, 1); + colInfoDataEnsureCapacity(out->columnData, 1); int32_t code = vectorConvertImpl(&in, out); sclFreeParam(&in); @@ -193,7 +193,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SValueNode *valueNode = (SValueNode *)node; param->numOfRows = 1; - param->columnData = createColumnInfoData(&valueNode->node.resType, 1); + param->columnData = sclCreateColumnInfoData(&valueNode->node.resType, 1); if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) { colDataAppendNULL(param->columnData, 0); } else { @@ -245,7 +245,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index); - if (NULL == block || ref->slotId >= block->info.numOfCols) { + if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) { sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -431,7 +431,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_JRET(code); } - output->columnData = createColumnInfoData(&node->node.resType, rowNum); + output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); if (output->columnData == NULL) { sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -484,7 +484,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o output->numOfRows = rowNum; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - output->columnData = createColumnInfoData(&t, rowNum); + output->columnData = sclCreateColumnInfoData(&t, rowNum); if (output->columnData == NULL) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -539,7 +539,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp int32_t code = 0; SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); - output->columnData = createColumnInfoData(&node->node.resType, rowNum); + output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); if (output->columnData == NULL) { sclError("calloc failed, size:%d", (int32_t)rowNum * node->node.resType.bytes); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -901,7 +901,7 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } - colDataAssign(col, res->columnData, res->numOfRows); + colDataAssign(col, res->columnData, res->numOfRows, NULL); block->info.rows = res->numOfRows; sclFreeParam(res); @@ -983,7 +983,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - colDataAssign(pDst->columnData, res->columnData, res->numOfRows); + colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); pDst->numOfRows = res->numOfRows; taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 2369779105..1c5a12687d 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -120,7 +120,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu } default: { - colDataAssign(pOutputData, pInputData, pInput->numOfRows); + colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL); } } @@ -273,7 +273,7 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP } default: { - colDataAssign(pOutputData, pInputData, pInput->numOfRows); + colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL); } } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index bff49c2ca9..a8a095a1cf 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -837,7 +837,7 @@ int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int3 SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; output->numOfRows = input->numOfRows; - output->columnData = createColumnInfoData(&t, input->numOfRows); + output->columnData = sclCreateColumnInfoData(&t, input->numOfRows); if (output->columnData == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -908,7 +908,7 @@ static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SSca pDest->numOfRows = pParam->numOfRows; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - pDest->columnData = createColumnInfoData(&t, pParam->numOfRows); + pDest->columnData = sclCreateColumnInfoData(&t, pParam->numOfRows); if (pDest->columnData == NULL) { sclError("malloc %d failed", (int32_t)(pParam->numOfRows * sizeof(double))); return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 7fb1ffbd64..ada5877d48 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -105,29 +105,14 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 3; - res->info.rows = rowNum; - res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + SSDataBlock *res = createDataBlock(); for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = TSDB_DATA_TYPE_NULL; - idata.info.bytes = 10; - idata.info.colId = i + 1; - - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); + SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, 1+i); + blockDataAppendColInfo(res, &idata); } - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 3; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3); + blockDataAppendColInfo(res, &idata); blockDataEnsureCapacity(res, rowNum); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); @@ -142,23 +127,18 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in rnode->slotId = 2; rnode->colId = 3; + res->info.rows = rowNum; *block = res; } else { SSDataBlock *res = *block; int32_t idx = taosArrayGetSize(res->pDataBlock); - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 1 + idx; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - res->info.numOfCols++; + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1+idx); + blockDataAppendColInfo(res, &idata); + blockDataEnsureCapacity(res, rowNum); + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - - colInfoDataEnsureCapacity(pColumn, 0, rowNum); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -399,7 +379,7 @@ TEST(columnTest, smallint_column_greater_double_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, true); - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -407,9 +387,9 @@ TEST(columnTest, smallint_column_greater_double_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -430,7 +410,7 @@ TEST(columnTest, int_column_greater_smallint_value) { flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); @@ -454,7 +434,7 @@ TEST(columnTest, int_column_greater_smallint_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, false); - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -462,9 +442,9 @@ TEST(columnTest, int_column_greater_smallint_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -478,11 +458,11 @@ TEST(columnTest, int_column_in_double_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; int32_t leftv[5] = {1, 2, 3, 4, 5}; double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; - bool eRes[5] = {true, true, true, false, false}; - SSDataBlock *src = NULL; + bool eRes[5] = {true, true, true, false, false}; + SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); nodesListAppend(list, pRight); @@ -498,7 +478,7 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -506,9 +486,9 @@ TEST(columnTest, int_column_in_double_list) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -521,8 +501,8 @@ TEST(columnTest, int_column_in_double_list) { TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; - bool eRes[5] = {true, true, false, false, false}; - SSDataBlock *src = NULL; + bool eRes[5] = {true, true, false, false, false}; + SSDataBlock *src = NULL; SScalarParam res = {0}; char leftv[5][5]= {0}; char rightv[3][5]= {0}; @@ -531,22 +511,22 @@ TEST(columnTest, binary_column_in_binary_list) { leftv[i][3] = 'b' + i; leftv[i][4] = '0' + i; varDataSetLen(leftv[i], 3); - } + } for (int32_t i = 0; i < 2; ++i) { rightv[i][2] = 'a' + i; rightv[i][3] = 'b' + i; rightv[i][4] = '0' + i; varDataSetLen(rightv[i], 3); - } + } for (int32_t i = 2; i < 3; ++i) { rightv[i][2] = 'a' + i; rightv[i][3] = 'a' + i; rightv[i][4] = 'a' + i; varDataSetLen(rightv[i], 3); } - + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); nodesListAppend(list, pRight); @@ -556,13 +536,13 @@ TEST(columnTest, binary_column_in_binary_list) { nodesListAppend(list, pRight); flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -570,9 +550,9 @@ TEST(columnTest, binary_column_in_binary_list) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -586,31 +566,31 @@ TEST(columnTest, binary_column_like_binary) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; char rightv[64] = {0}; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {true, false, true, false, true}; - + bool eRes[5] = {true, false, true, false, true}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = 'a'; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); sprintf(&rightv[2], "%s", "__0"); varDataSetLen(rightv, strlen(&rightv[2])); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -618,9 +598,9 @@ TEST(columnTest, binary_column_like_binary) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -633,31 +613,31 @@ TEST(columnTest, binary_column_like_binary) { TEST(columnTest, binary_column_is_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {false, false, true, false, true}; - + bool eRes[5] = {false, false, true, false, true}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); colDataAppend(pcolumn, 2, NULL, true); colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -665,9 +645,9 @@ TEST(columnTest, binary_column_is_null) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -680,31 +660,31 @@ TEST(columnTest, binary_column_is_null) { TEST(columnTest, binary_column_is_not_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {true, true, true, true, false}; - + bool eRes[5] = {true, true, true, true, false}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -712,9 +692,9 @@ TEST(columnTest, binary_column_is_not_null) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -735,13 +715,13 @@ TEST(opTest, smallint_column_greater_int_column) { flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -749,9 +729,9 @@ TEST(opTest, smallint_column_greater_int_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -773,13 +753,13 @@ TEST(opTest, smallint_value_add_int_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -787,9 +767,9 @@ TEST(opTest, smallint_value_add_int_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -816,13 +796,13 @@ TEST(opTest, bigint_column_multi_binary_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -830,9 +810,9 @@ TEST(opTest, bigint_column_multi_binary_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -859,13 +839,13 @@ TEST(opTest, smallint_column_and_binary_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -873,9 +853,9 @@ TEST(opTest, smallint_column_and_binary_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -897,13 +877,13 @@ TEST(opTest, smallint_column_or_float_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -911,9 +891,9 @@ TEST(opTest, smallint_column_or_float_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -935,13 +915,13 @@ TEST(opTest, smallint_column_or_double_value) { flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -949,9 +929,9 @@ TEST(opTest, smallint_column_or_double_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, true); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -964,28 +944,28 @@ TEST(opTest, smallint_column_or_double_value) { TEST(opTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {false, true, false, true, false}; - + bool eRes[5] = {false, true, false, true, false}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -993,9 +973,9 @@ TEST(opTest, binary_column_is_true) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1027,12 +1007,12 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1042,7 +1022,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); @@ -1050,13 +1030,13 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1064,9 +1044,9 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1096,13 +1076,13 @@ TEST(filterModelogicTest, same_column_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); - flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); @@ -1111,7 +1091,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); @@ -1119,13 +1099,13 @@ TEST(filterModelogicTest, same_column_and_or_and) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1133,9 +1113,9 @@ TEST(filterModelogicTest, same_column_and_or_and) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1165,12 +1145,12 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1180,7 +1160,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1188,13 +1168,13 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1202,9 +1182,9 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1234,12 +1214,12 @@ TEST(filterModelogicTest, same_column_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1249,7 +1229,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1257,13 +1237,13 @@ TEST(filterModelogicTest, same_column_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1271,9 +1251,9 @@ TEST(filterModelogicTest, same_column_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1305,12 +1285,12 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1320,7 +1300,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1328,13 +1308,13 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1342,7 +1322,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); for (int32_t i = 0; i < rowNum; ++i) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index d879150200..a6b7de71be 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -84,30 +84,25 @@ void scltInitLogFile() { void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { if (newBlock) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 1; - res->info.rows = rows; - res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + SSDataBlock *res = createDataBlock(); + SColumnInfoData idata = {0}; idata.info = *colInfo; - - taosArrayPush(res->pDataBlock, &idata); - taosArrayPush(pBlockList, &res); + blockDataAppendColInfo(res, &idata); blockDataEnsureCapacity(res, rows); + res->info.rows = rows; + + taosArrayPush(pBlockList, &res); *dataBlockId = taosArrayGetSize(pBlockList) - 1; res->info.blockId = *dataBlockId; *slotId = 0; } else { SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); - res->info.numOfCols++; SColumnInfoData idata = {0}; idata.info = *colInfo; - - colInfoDataEnsureCapacity(&idata, 0, rows); - - taosArrayPush(res->pDataBlock, &idata); + blockDataAppendColInfo(res, &idata); *dataBlockId = taosArrayGetSize(pBlockList) - 1; *slotId = taosArrayGetSize(res->pDataBlock) - 1; @@ -144,29 +139,18 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 3; + SSDataBlock *res = createDataBlock(); res->info.rows = rowNum; - res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = TSDB_DATA_TYPE_NULL; - idata.info.bytes = 10; - idata.info.colId = i + 1; - + SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, i + 1); int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); + blockDataAppendColInfo(res, &idata); } - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 3; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - colInfoDataEnsureCapacity(&idata, 0, rowNum); - taosArrayPush(res->pDataBlock, &idata); + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3); + blockDataAppendColInfo(res, &idata); + blockDataEnsureCapacity(res, rowNum); + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -185,17 +169,11 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SSDataBlock *res = *block; int32_t idx = taosArrayGetSize(res->pDataBlock); - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 1 + idx; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - res->info.numOfCols++; - SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1 + idx); + blockDataAppendColInfo(res, &idata); + blockDataEnsureCapacity(res, rowNum); - colInfoDataEnsureCapacity(pColumn, 0, rowNum); + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -2033,7 +2011,7 @@ void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t input->numOfRows = num; input->columnData->info = createColumnInfo(0, type, bytes); - colInfoDataEnsureCapacity(input->columnData, 0, num); + colInfoDataEnsureCapacity(input->columnData, num); if (setVal) { for (int32_t i = 0; i < num; ++i) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 1894f697c0..78179cb71e 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -74,10 +74,12 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; pRetrieve->numOfRows = htonl(pBlock->info.rows); - pRetrieve->numOfCols = htonl(pBlock->info.numOfCols); + + int32_t numOfCols = (int32_t) taosArrayGetSize(pBlock->pDataBlock); + pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); From a1401db4372f791a968d32d3a91e6477bee1b3b7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 17:23:38 +0800 Subject: [PATCH 02/55] fix(query): fix syntax error in unit test. --- source/libs/scalar/test/filter/filterTests.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index ada5877d48..8f9332cef8 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -921,7 +921,7 @@ TEST(opTest, smallint_column_or_double_value) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -965,7 +965,7 @@ TEST(opTest, binary_column_is_true) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1036,7 +1036,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1105,7 +1105,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1174,7 +1174,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1243,7 +1243,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1314,7 +1314,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); From aa4d86359844212f6f1051706cd73afdb21cbedd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 17:36:48 +0800 Subject: [PATCH 03/55] fix(query): fix syntax error. --- source/libs/command/src/command.c | 38 +++++++------------------ source/libs/executor/src/sortoperator.c | 2 +- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 91c3628db2..f8aa7ead8c 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -114,22 +114,12 @@ static int32_t execResetQueryCache() { return catalogClearCache(); } static SSDataBlock* buildCreateDBResultDataBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = SHOW_CREATE_DB_RESULT_COLS; - pBlock->info.hasVarCol = true; - - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD1_LEN; - - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD2_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); + SSDataBlock* pBlock = createDataBlock(); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1); + blockDataAppendColInfo(pBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); return pBlock; } @@ -271,21 +261,13 @@ static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveT } static SSDataBlock* buildCreateTbResultDataBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = SHOW_CREATE_TB_RESULT_COLS; - pBlock->info.hasVarCol = true; + SSDataBlock* pBlock = createDataBlock(); - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD1_LEN, 1); + blockDataAppendColInfo(pBlock, &infoData); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD1_LEN; - - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD2_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); return pBlock; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index a69f11f51d..3d8de1c1fd 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -286,7 +286,7 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; From c79eda9f707fc29dada5e8c6a515ddcfcbb5b698 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 19:34:56 +0800 Subject: [PATCH 04/55] fix(query): prepare the buffer before serialize the retrieved data. --- source/common/src/tdatablock.c | 65 +++++-------------------- source/libs/executor/src/executorimpl.c | 9 ++-- source/libs/executor/src/scanoperator.c | 3 +- 3 files changed, 16 insertions(+), 61 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7856f3a29b..b98001cb03 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -302,50 +302,20 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p ASSERT(pBlockInfo->capacity >= numOfRows); if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { -// if (pColumnInfoData->capacity < numOfRows) { -// char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); -// if (p == NULL) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// -// memset(p, 0, sizeof(int32_t)); -// pColumnInfoData->capacity = numOfRows; -// pColumnInfoData->varmeta.offset = (int32_t*)p; -// } - memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); -// if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { -// char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); -// if (tmp == NULL) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// -// pColumnInfoData->pData = tmp; -// pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; -// } + if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumnInfoData->pData = tmp; + pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; + } - memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); pColumnInfoData->varmeta.length = pSource->varmeta.length; + memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); } else { -// if (pColumnInfoData->capacity < numOfRows) { -// char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); -// if (tmp == NULL) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// -// memset(tmp, 0, BitmapLen(numOfRows)); -// pColumnInfoData->nullbitmap = tmp; -// -// int32_t newSize = numOfRows * pColumnInfoData->info.bytes; -// tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); -// if (tmp == NULL) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// -// pColumnInfoData->pData = tmp; -// pColumnInfoData->capacity = numOfRows; -// } - memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); } @@ -1874,8 +1844,6 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen } const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { - blockDataEnsureCapacity(pBlock, numOfRows); - const char* pStart = pData; int32_t dataLen = *(int32_t*)pStart; @@ -1884,11 +1852,6 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t pBlock->info.groupId = *(uint64_t*)pStart; pStart += sizeof(uint64_t); - if (pBlock->pDataBlock == NULL) { - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - taosArraySetSize(pBlock->pDataBlock, numOfCols); - } - for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData->info.type = *(int16_t*)pStart; @@ -1919,22 +1882,16 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); pStart += sizeof(int32_t) * numOfRows; - if (colLen[i] > 0) { + if (colLen[i] > 0 && pColInfoData->pData == NULL) { taosMemoryFreeClear(pColInfoData->pData); pColInfoData->pData = taosMemoryMalloc(colLen[i]); } } else { - if (pColInfoData->nullbitmap == NULL) { - pColInfoData->nullbitmap = taosMemoryCalloc(1, BitmapLen(numOfRows)); - } memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); pStart += BitmapLen(numOfRows); } if (colLen[i] > 0) { - if (pColInfoData->pData == NULL) { - pColInfoData->pData = taosMemoryCalloc(1, colLen[i]); - } memcpy(pColInfoData->pData, pStart, colLen[i]); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0c6e7fe5f6..9f917c9b81 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2075,14 +2075,13 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo blockCompressDecode(pBlock, numOfCols, numOfRows, pStart); blockDataEnsureCapacity(pRes, numOfRows); - relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - // data from mnode pRes->info.rows = numOfRows; + relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - taosArrayDestroy(pBlock->pDataBlock); - taosMemoryFree(pBlock); - // blockDataDestroy(pBlock); +// taosArrayDestroy(pBlock->pDataBlock); +// taosMemoryFree(pBlock); + blockDataDestroy(pBlock); } // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 49e821a71a..156be6ba13 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1366,7 +1366,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; const char* db = NULL; @@ -1558,7 +1557,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { if (pRsp->numOfRows == 0 || pRsp->completed) { pOperator->status = OP_EXEC_DONE; - qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64 " ", GET_TASKID(pTaskInfo), + qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), pRsp->numOfRows, pInfo->loadInfo.totalRows); if (pRsp->numOfRows == 0) { From b097e78708e41926175f2a8478168461724fe886 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 19:40:59 +0800 Subject: [PATCH 05/55] fix(query): clear buffer before set data. --- source/common/src/tdatablock.c | 2 +- source/libs/executor/src/executorimpl.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b98001cb03..4a9d573881 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1201,7 +1201,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { pBlock->info.rows = 0; pBlock->info.capacity = 0; - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData colInfo = {.hasNull = true, .info = p->info}; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9f917c9b81..db107d444b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2047,6 +2047,8 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList) { if (pColList == NULL) { // data from other sources + blockDataCleanup(pRes); + blockDataEnsureCapacity(pRes, numOfRows); blockCompressDecode(pRes, numOfOutput, numOfRows, pData); pRes->info.rows = numOfRows; } else { // extract data according to pColList From 0a884d58bbc052482ec3c4ea73935772b515607d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 19:41:57 +0800 Subject: [PATCH 06/55] refactor: remove unnecessary code. --- source/libs/executor/src/executorimpl.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index db107d444b..bd99fd0be3 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2050,7 +2050,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo blockDataCleanup(pRes); blockDataEnsureCapacity(pRes, numOfRows); blockCompressDecode(pRes, numOfOutput, numOfRows, pData); - pRes->info.rows = numOfRows; } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; @@ -2080,9 +2079,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo // data from mnode pRes->info.rows = numOfRows; relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - -// taosArrayDestroy(pBlock->pDataBlock); -// taosMemoryFree(pBlock); blockDataDestroy(pBlock); } From 72a942b09ec311556748b06518ab2a91df7f160b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 19:44:14 +0800 Subject: [PATCH 07/55] refactor(query): do internal refactor. --- include/common/tdatablock.h | 2 +- source/common/src/tdatablock.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index d65fdc0675..7e9dd43bae 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -183,7 +183,7 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u int32_t getJsonValueLen(const char* data); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4a9d573881..5d6dde04ca 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -214,7 +214,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } } -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRow2 == 0) { From 324eecdd998d9527d83f98f32678bc3a8abc7047 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 20:24:26 +0800 Subject: [PATCH 08/55] fix(query): prepare the buffer before assigning result. --- source/common/src/tdatablock.c | 4 +++- source/libs/scalar/src/scalar.c | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5d6dde04ca..57d75770cf 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -299,7 +299,9 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p return numOfRows; } - ASSERT(pBlockInfo->capacity >= numOfRows); + if (pBlockInfo != NULL) { + ASSERT(pBlockInfo->capacity >= numOfRows); + } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 9890cff6f2..75a9bc3809 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -982,7 +982,8 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + + colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows); colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); pDst->numOfRows = res->numOfRows; taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); From ed8002f903f469636ce69822e1a7fccd98045db0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 20:34:24 +0800 Subject: [PATCH 09/55] fix(query): fix syntax error. --- source/libs/stream/src/streamDispatch.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index d26100d455..b5a9e221a7 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -96,16 +96,17 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve = taosMemoryCalloc(1, dataStrLen); if (pRetrieve == NULL) return -1; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); pRetrieve->useconds = 0; pRetrieve->precision = TSDB_DEFAULT_PRECISION; pRetrieve->compressed = 0; pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; pRetrieve->numOfRows = htonl(pBlock->info.rows); - pRetrieve->numOfCols = htonl(pBlock->info.numOfCols); + pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); SStreamRetrieveReq req = { .streamId = pTask->streamId, From 02a53ee81f8fb635035f51682e6a80f47ab9686d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 21:03:59 +0800 Subject: [PATCH 10/55] fix(query): set correct memory buffer size. --- source/common/src/tdatablock.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 57d75770cf..4f513335eb 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1169,7 +1169,9 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { return TSDB_CODE_SUCCESS; } - pDataBlock->info.capacity = numOfRows; + if (pDataBlock->info.capacity < numOfRows) { + pDataBlock->info.capacity = numOfRows; + } size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { @@ -1878,15 +1880,21 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - pColInfoData->varmeta.length = colLen[i]; - pColInfoData->varmeta.allocLen = colLen[i]; +// pColInfoData->varmeta.length = colLen[i]; +// pColInfoData->varmeta.allocLen = colLen[i]; memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); pStart += sizeof(int32_t) * numOfRows; - if (colLen[i] > 0 && pColInfoData->pData == NULL) { - taosMemoryFreeClear(pColInfoData->pData); - pColInfoData->pData = taosMemoryMalloc(colLen[i]); + if (colLen[i] > 0 && pColInfoData->varmeta.allocLen < colLen[i]) { + char* tmp = taosMemoryRealloc(pColInfoData->pData, colLen[i]); + if (tmp == NULL) { + return NULL; + } + + pColInfoData->pData = tmp; + pColInfoData->varmeta.allocLen = colLen[i]; + pColInfoData->varmeta.length = colLen[i]; } } else { memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); From ecc05bf75c540a1cccd34682659ab1858da46325 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 21:12:08 +0800 Subject: [PATCH 11/55] fix(query): check pointer before use it. --- source/common/src/tdatablock.c | 3 --- source/libs/function/src/builtinsimpl.c | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4f513335eb..168d949631 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1880,9 +1880,6 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { -// pColInfoData->varmeta.length = colLen[i]; -// pColInfoData->varmeta.allocLen = colLen[i]; - memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); pStart += sizeof(int32_t) * numOfRows; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4d869705ad..b0de3b3364 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1450,8 +1450,8 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple int32_t pageId = pTuplePos->pageId; int32_t offset = pTuplePos->offset; - int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock); if (pTuplePos->pageId != -1) { + int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock); SFilePage* pPage = getBufPage(pCtx->pBuf, pageId); bool* nullList = (bool*)((char*)pPage + offset); From 300f35188e0c2887bb6d25f5805e131a29f80733 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 21:18:11 +0800 Subject: [PATCH 12/55] fix(query): prepare the SColumnInfoData. --- source/common/src/tdatablock.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 168d949631..93d3661caf 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1856,6 +1856,11 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t pBlock->info.groupId = *(uint64_t*)pStart; pStart += sizeof(uint64_t); + if (pBlock->pDataBlock == NULL) { + pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + taosArraySetSize(pBlock->pDataBlock, numOfCols); + } + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData->info.type = *(int16_t*)pStart; From d3d258caa9d791d0e103b13026a029a7cc5938c1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 23:05:34 +0800 Subject: [PATCH 13/55] fix(query): prepare buffer before copying data. --- source/libs/executor/src/sortoperator.c | 8 +++++--- source/libs/function/src/tudf.c | 1 - tests/script/general/parser/udf.sim | 1 + tests/script/tsim/query/udf.sim | 1 + 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 3d8de1c1fd..79d5fe104f 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -120,6 +120,8 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i } if (p->info.rows > 0) { + blockDataEnsureCapacity(pDataBlock, capacity); + // todo extract function to handle this int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { @@ -132,7 +134,6 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i } pDataBlock->info.rows = p->info.rows; - pDataBlock->info.capacity = p->info.rows; } blockDataDestroy(p); @@ -258,6 +259,7 @@ typedef struct SGroupSortOperatorInfo { SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SGroupSortOperatorInfo* pInfo) { blockDataCleanup(pDataBlock); + blockDataEnsureCapacity(pDataBlock, capacity); SSDataBlock* p = tsortGetSortedDataBlock(pHandle); if (p == NULL) { @@ -571,7 +573,8 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData } } - if (p->info.rows > 0) { + if (p->info.rows > 0) {// todo extract method + blockDataEnsureCapacity(pDataBlock, p->info.rows); int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); @@ -583,7 +586,6 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData } pDataBlock->info.rows = p->info.rows; - pDataBlock->info.capacity = p->info.rows; pDataBlock->info.groupId = pInfo->groupId; } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index ea2a7567e4..aead5f2867 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1081,7 +1081,6 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; - SSDataBlock* pTempBlock = createDataBlock(); pTempBlock->info.rows = pInput->totalRows; pTempBlock->info.uid = pInput->uid; diff --git a/tests/script/general/parser/udf.sim b/tests/script/general/parser/udf.sim index c048a7c37a..4b767dbf60 100644 --- a/tests/script/general/parser/udf.sim +++ b/tests/script/general/parser/udf.sim @@ -44,6 +44,7 @@ endi sql create function n02 as '/tmp/normal' outputtype bool; sql show functions; if $rows != 1 then + print expect 1, actual $rows return -1 endi diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index d9821a0495..4e95095172 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -44,6 +44,7 @@ endi sql select udf2(f) from t; if $rows != 1 then + print expect 1, actual $rows return -1 endi if $data00 != 2.236067977 then From 32e479858fe6de8ac0671aa2f24c9491367f186b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 23:48:23 +0800 Subject: [PATCH 14/55] fix(query): disable type check. --- source/common/src/tdatablock.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 93d3661caf..0fab1c4f5a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1268,7 +1268,8 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoDat return terrno; } - ASSERT(pColInfoData->info.type != 0); + // todo disable it temporarily +// ASSERT(pColInfoData->info.type != 0); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { pBlock->info.hasVarCol = true; } From a0795fdd10b7794f4bd9dede1c97f0bfab950b46 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 00:47:28 +0800 Subject: [PATCH 15/55] fix(query): set correct varchar length. --- source/common/src/tdatablock.c | 3 ++- source/libs/executor/src/executorimpl.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 0fab1c4f5a..565f8da0a0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1897,8 +1897,9 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t pColInfoData->pData = tmp; pColInfoData->varmeta.allocLen = colLen[i]; - pColInfoData->varmeta.length = colLen[i]; } + + pColInfoData->varmeta.length = colLen[i]; } else { memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); pStart += BitmapLen(numOfRows); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ed867bed45..188edaa68f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2048,7 +2048,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo SArray* pColList) { if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); - blockDataEnsureCapacity(pRes, numOfRows); +// blockDataEnsureCapacity(pRes, numOfRows); blockCompressDecode(pRes, numOfOutput, numOfRows, pData); } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); From 673261a1bd1dfbdd9d7c85df41371c883f5f606b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 09:42:29 +0800 Subject: [PATCH 16/55] fix(query): set correct varchar length. --- source/libs/executor/src/scanoperator.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b978d54a5d..e1b9e7e3de 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2005,6 +2005,7 @@ _error: return code; } +// todo refactor or remove it static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2145,7 +2146,13 @@ static SSDataBlock* getTableDataBlock(void* param) { continue; } - tsdbRetrieveDataBlockInfo(reader, &pBlock->info); + blockDataCleanup(pBlock); + SDataBlockInfo binfo = pBlock->info; + tsdbRetrieveDataBlockInfo(reader, &binfo); + + binfo.capacity = binfo.rows; + blockDataEnsureCapacity(pBlock, binfo.capacity); + pBlock->info = binfo; uint32_t status = 0; int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status); From 0457fab0779f9ab5bc3c393cb1a1b191f6e270fb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 10:03:43 +0800 Subject: [PATCH 17/55] fix(query): adjust the capacity of ssdatablock. --- include/common/tcommon.h | 2 +- source/libs/executor/src/scanoperator.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 9c0c3c0616..f841a7d4cd 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -68,7 +68,7 @@ typedef struct SDataBlockInfo { STimeWindow window; int32_t rows; int32_t rowSize; - int64_t uid; // the uid of table, from which current data block comes + uint64_t uid; // the uid of table, from which current data block comes uint16_t blockId; // block id, generated by physical planner uint64_t groupId; // no need to serialize int16_t hasVarCol; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e1b9e7e3de..b1bd50daf0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -969,6 +969,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { uint64_t uid = 0; int32_t numOfRows = 0; + // todo refactor int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows); if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { @@ -976,12 +977,11 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { return NULL; } - blockDataEnsureCapacity(pInfo->pRes, numOfRows); - pInfo->pRes->info.groupId = groupId; pInfo->pRes->info.rows = numOfRows; pInfo->pRes->info.uid = uid; pInfo->pRes->info.type = STREAM_NORMAL; + pInfo->pRes->info.capacity = numOfRows; // for generating rollup SMA result, each time is an independent time serie. // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this From ce7e7fe23d64fb75ab1d885e759a71c0e07be02c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 10:13:35 +0800 Subject: [PATCH 18/55] refactor:remove some debug info. --- source/libs/executor/src/executor.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e805de3727..cf72dfd796 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -68,8 +68,6 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayClear(p->pDataBlock); taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock); - printf("size------------->%ld, %ld, %p, \n", taosArrayGetSize(pDataBlock->pDataBlock), taosArrayGetSize(pDataBlock->pDataBlock) * pDataBlock->pDataBlock->elemSize, - pDataBlock->pDataBlock); taosArrayPush(pInfo->pBlockLists, &p); } } else { From 4576b0a94d7d2a30224d893b5fbb2fd2d18bb28e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 11:07:14 +0800 Subject: [PATCH 19/55] fix(query): fix syntax error. --- source/libs/executor/inc/executorimpl.h | 4 ++-- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/scanoperator.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 9bcfae549d..ccdbe5c890 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -838,8 +838,8 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, - SNode* pTagCond); + STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); + SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ad5ebb5974..cd3b8cb705 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4009,7 +4009,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; - createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId, pTagCond); + createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); STableScanInfo* pScanInfo = pOperator->info; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d1d0a70115..2e3ae851bf 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1956,7 +1956,7 @@ int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { } int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, SNode* pTagCond) { + STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) { int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { return code; From 91eb4ff88b51f4ed88ae5031df52512676f54efb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 11:32:21 +0800 Subject: [PATCH 20/55] test(query): update test case. --- tests/system-test/2-query/json_tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index ff7adfc759..6cd3f49636 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -509,7 +509,7 @@ class TDTestCase: tdSql.checkData(0, 0, 5.3) # tdSql.query("select twa(dataint) from jsons1 where jtag is not null") # tdSql.checkData(0, 0, 36) - tdSql.error("select irate(dataint) from jsons1 where jtag is not null") + tdSql.query("select irate(dataint) from jsons1 where jtag is not null") tdSql.query("select sum(dataint) from jsons1 where jtag->'tag1' is not null") tdSql.checkData(0, 0, 45) tdSql.query("select stddev(dataint) from jsons1 where jtag->'tag1'>1") From a31569f116b8bc22266a9ff7eb3e4b73c70468a3 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 13:38:20 +0800 Subject: [PATCH 21/55] fix: set correct datablock rows and columns --- source/common/src/tdatablock.c | 2 +- source/libs/function/src/tudf.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 565f8da0a0..40e21fc6ac 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -463,7 +463,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 } pDst->info = pBlock->info; - + pDst->info.rows = 0; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index aead5f2867..da9474ede0 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -763,6 +763,7 @@ void freeUdfInterBuf(SUdfInterBuf *buf) { int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) { udfBlock->numOfRows = block->info.rows; + udfBlock->numOfCols = taosArrayGetSize(block->pDataBlock); udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn*)); for (int32_t i = 0; i < udfBlock->numOfCols; ++i) { udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn)); From 044f27bbca0746007f20ab900b7711fef1493254 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 13:58:36 +0800 Subject: [PATCH 22/55] fix(query): avg/stddev divide 0 handling TD-16762 --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index d7ff03ec59..647f039d96 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -927,7 +927,7 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { // check for overflow if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) { - GET_RES_INFO(pCtx)->isNullRes = 1; + GET_RES_INFO(pCtx)->numOfRes = 0; } return functionFinalize(pCtx, pBlock); From edc42a78abc85235f8f53e9025062db19c1e9a9a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 13:59:31 +0800 Subject: [PATCH 23/55] fix: fix compilation error --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 12 ++++++------ source/libs/command/src/command.c | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 9c8918a445..b9bd49516a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -203,7 +203,6 @@ SSDataBlock* dmBuildVariablesBlock(void) { taosArrayPush(pBlock->pDataBlock, &colInfoData); } - pBlock->info.numOfCols = pMeta[index].colNum; pBlock->info.hasVarCol = true; return pBlock; @@ -262,8 +261,9 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId); - size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pBlock->info.numOfCols + - blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols + + blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(numOfCols); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { @@ -274,10 +274,10 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } char *pStart = pRsp->data; - *(int32_t *)pStart = htonl(pBlock->info.numOfCols); + *(int32_t *)pStart = htonl(numOfCols); pStart += sizeof(int32_t); // number of columns - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SSysTableSchema *pSchema = (SSysTableSchema *)pStart; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); @@ -289,7 +289,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t len = 0; - blockCompressEncode(pBlock, pStart, &len, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pStart, &len, numOfCols, false); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 2ba0f1a77d..8d01ed7315 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -524,10 +524,9 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return TSDB_CODE_FAILED; static SSDataBlock* buildLocalVariablesResultDataBlock() { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = SHOW_LOCAL_VARIABLES_RESULT_COLS; pBlock->info.hasVarCol = true; - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); + pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); SColumnInfoData infoData = {0}; infoData.info.type = TSDB_DATA_TYPE_VARCHAR; From ca4d7329ce971984b854506e538bbe9583302356 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 23 Jun 2022 14:07:18 +0800 Subject: [PATCH 24/55] refactor(sync): modify wal error log --- source/libs/sync/src/syncAppendEntries.c | 27 +++-------- source/libs/sync/src/syncAppendEntriesReply.c | 42 ++++++++--------- source/libs/sync/src/syncMain.c | 5 +- source/libs/sync/src/syncRaftLog.c | 46 ++++++++++++++----- 4 files changed, 64 insertions(+), 56 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index bc5342cc7e..31176d942e 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -466,14 +466,11 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs int32_t code = 0; // print log - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntries, vgId:%d, term:%lu", ths->vgId, - ths->pRaftStore->currentTerm); - syncAppendEntriesLog2(logBuf, pMsg); + syncAppendEntriesLog2("==syncNodeOnAppendEntriesSnapshotCb==", pMsg); // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { - sInfo("recv SyncAppendEntries maybe replica already dropped"); + syncNodeEventLog(ths, "recv sync-append-entries, maybe replica already dropped"); return ret; } @@ -497,7 +494,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs do { bool condition = pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE; if (condition) { - sTrace("recv SyncAppendEntries, candidate to follower"); + syncNodeEventLog(ths, "recv sync-append-entries, candidate to follower"); syncNodeBecomeFollower(ths, "from candidate by append entries"); // do not reply? @@ -538,11 +535,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = condition1 || condition2 || condition3 || condition4; if (condition) { - sTrace( - "recv SyncAppendEntries, fake match, myLastIndex:%ld, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, " - "condition1:%d, condition2:%d, condition3:%d, condition4:%d", - myLastIndex, ths->pLogStore->syncLogBeginIndex(ths->pLogStore), - ths->pLogStore->syncLogEndIndex(ths->pLogStore), condition1, condition2, condition3, condition4); + syncNodeEventLog(ths, "recv sync-append-entries, fake match"); // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); @@ -576,8 +569,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && (pMsg->prevLogIndex <= ths->commitIndex); if (condition) { - sTrace("recv SyncAppendEntries, fake match2, msg-prevLogIndex:%ld, my-commitIndex:%ld", pMsg->prevLogIndex, - ths->commitIndex); + syncNodeEventLog(ths, "recv sync-append-entries, fake match2"); SyncIndex matchIndex = ths->commitIndex; bool hasAppendEntries = pMsg->dataLen > 0; @@ -650,11 +642,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = condition1 || condition2; if (condition) { - sTrace( - "recv SyncAppendEntries, not match, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, condition1:%d, " - "condition2:%d, logOK:%d", - ths->pLogStore->syncLogBeginIndex(ths->pLogStore), ths->pLogStore->syncLogEndIndex(ths->pLogStore), - condition1, condition2, logOK); + syncNodeEventLog(ths, "recv sync-append-entries, not match"); // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); @@ -693,8 +681,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs // has entries in SyncAppendEntries msg bool hasAppendEntries = pMsg->dataLen > 0; - sTrace("recv SyncAppendEntries, match, myLastIndex:%ld, hasExtraEntries:%d, hasAppendEntries:%d", myLastIndex, - hasExtraEntries, hasAppendEntries); + syncNodeEventLog(ths, "recv sync-append-entries, match"); if (hasExtraEntries) { // make log same, rollback deleted entries diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index f13a3604da..6ca356b4f6 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -101,32 +101,27 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries int32_t ret = 0; // print log - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, vgId:%d, term:%lu", ths->vgId, - ths->pRaftStore->currentTerm); - syncAppendEntriesReplyLog2(logBuf, pMsg); + syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplySnapshotCb==", pMsg); // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { - sInfo("recv SyncAppendEntriesReply, maybe replica already dropped"); - return ret; + syncNodeEventLog(ths, "recv sync-append-entries-reply, maybe replica already dropped"); + return 0; } // drop stale response if (pMsg->term < ths->pRaftStore->currentTerm) { - sTrace("recv SyncAppendEntriesReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, - ths->pRaftStore->currentTerm); - return ret; + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%lu, drop stale response", pMsg->term); + syncNodeEventLog(ths, logBuf); + return 0; } - syncIndexMgrLog2("recv SyncAppendEntriesReply, before pNextIndex:", ths->pNextIndex); - syncIndexMgrLog2("recv SyncAppendEntriesReply, before pMatchIndex:", ths->pMatchIndex); if (gRaftDetailLog) { - SSnapshot snapshot; - ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); - sTrace("recv SyncAppendEntriesReply, before snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", - snapshot.lastApplyIndex, snapshot.lastApplyTerm); + syncNodeEventLog(ths, "recv sync-append-entries-reply, before"); } + syncIndexMgrLog2("recv sync-append-entries-reply, before pNextIndex:", ths->pNextIndex); + syncIndexMgrLog2("recv sync-append-entries-reply, before pMatchIndex:", ths->pMatchIndex); // no need this code, because if I receive reply.term, then I must have sent for that term. // if (pMsg->term > ths->pRaftStore->currentTerm) { @@ -134,12 +129,10 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries // } if (pMsg->term > ths->pRaftStore->currentTerm) { - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, error term, receive_term:%lu current_term:%lu", - pMsg->term, ths->pRaftStore->currentTerm); - syncNodeLog2(logBuf, ths); - sError("%s", logBuf); - return ret; + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%lu", pMsg->term); + syncNodeErrorLog(ths, logBuf); + return -1; } ASSERT(pMsg->term == ths->pRaftStore->currentTerm); @@ -228,8 +221,11 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } } - syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex); - syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex); + if (gRaftDetailLog) { + syncNodeEventLog(ths, "recv sync-append-entries-reply, after"); + } + syncIndexMgrLog2("recv sync-append-entries-reply, after pNextIndex:", ths->pNextIndex); + syncIndexMgrLog2("recv sync-append-entries-reply, after pMatchIndex:", ths->pMatchIndex); return ret; } \ No newline at end of file diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 732f450f4c..8979f244f0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -416,7 +416,7 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct } } sMeta->lastConfigIndex = lastIndex; - sTrace("vgId:%d, get snapshot meta by index:%" PRId64 " lastConfigIndex:%" PRId64, pSyncNode->vgId, snapshotIndex, + sTrace("vgId:%d, get snapshot meta by index:%" PRId64 " lcindex:%" PRId64, pSyncNode->vgId, snapshotIndex, sMeta->lastConfigIndex); taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -433,8 +433,9 @@ SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapsho lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i]; } } + sTrace("vgId:%d, sync get snapshot last config index, index:%ld lcindex:%ld", pSyncNode->vgId, snapshotLastApplyIndex, + lastIndex); - sTrace("sync syncNodeGetSnapshotConfigIndex index:%ld lastConfigIndex:%ld", snapshotLastApplyIndex, lastIndex); return lastIndex; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 701b4a7b93..f000c9f117 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -46,13 +46,13 @@ static SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); // refactor, log[0 .. n] ==> log[m .. n] static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex) { - sTrace("raftLogSetBeginIndex beginIndex:%ld", beginIndex); - // if beginIndex == 0, donot need call this funciton ASSERT(beginIndex > 0); SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; + sTrace("vgId:%d, reset wal begin index:%ld", pData->pSyncNode->vgId, beginIndex); + pData->beginIndex = beginIndex; walRestoreFromSnapshot(pWal, beginIndex - 1); return 0; @@ -160,8 +160,12 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, - pEntry->index, err, err, errStr, sysErr, sysErrStr); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + pEntry->index, err, err, errStr, sysErr, sysErrStr); + syncNodeErrorLog(pData->pSyncNode, logBuf); + ASSERT(0); } @@ -240,8 +244,17 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, index, - err, err, errStr, sysErr, sysErrStr); + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, err, + err, errStr, sysErr, sysErrStr); + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { + syncNodeEventLog(pData->pSyncNode, logBuf); + } else { + syncNodeErrorLog(pData->pSyncNode, logBuf); + } + } while (0); int32_t saveErr = terrno; walCloseReadHandle(pWalHandle); @@ -368,8 +381,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, - pEntry->index, err, err, errStr, sysErr, sysErrStr); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + pEntry->index, err, err, errStr, sysErr, sysErrStr); + syncNodeErrorLog(pData->pSyncNode, logBuf); ASSERT(0); } @@ -398,12 +414,20 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, - index, err, err, errStr, sysErr, sysErrStr); + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, + err, err, errStr, sysErr, sysErrStr); + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { + syncNodeEventLog(pData->pSyncNode, logBuf); + } else { + syncNodeErrorLog(pData->pSyncNode, logBuf); + } + } while (0); ASSERT(0); } - // ASSERT(walReadWithHandle(pWalHandle, index) == 0); SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); ASSERT(pEntry != NULL); From cfad058333c69398337643f259f099a5c4da79b0 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 13:58:36 +0800 Subject: [PATCH 25/55] fix(query): avg/stddev divide 0 handling TD-16762 --- source/libs/function/src/builtinsimpl.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 647f039d96..5f3d1b89db 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1774,6 +1774,11 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pStddevRes->result = sqrt(fabs(pStddevRes->quadraticDSum / ((double)pStddevRes->count) - avg * avg)); } + // check for overflow + if (isinf(pStddevRes->result) || isnan(pStddevRes->result)) { + GET_RES_INFO(pCtx)->numOfRes = 0; + } + return functionFinalize(pCtx, pBlock); } From c4b1d43bbc9f01ecdcd3045053a57a03e97f7ed6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 14:12:03 +0800 Subject: [PATCH 26/55] fix: show dnode 1 variables count --- tests/script/tsim/show/basic.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index 8f9362fc69..e2138e0008 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -233,7 +233,7 @@ if $rows != 4 then endi sql show dnode 1 variables; -if $rows != 114 then +if $rows != 116 then return -1 endi From d5a27d50e321406020d6cd88d8a5517fe0ad1e53 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 13:58:36 +0800 Subject: [PATCH 27/55] fix(query): avg/stddev divide 0 handling TD-16762 --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5f3d1b89db..690319a4cf 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -561,7 +561,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { // check for overflow if (IS_FLOAT_TYPE(type) && (isinf(pSumRes->dsum) || isnan(pSumRes->dsum))) { - GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElems = 0; } _sum_over: From e18329ac194ae0511359e0bd348fcf9133a1570a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 23 Jun 2022 14:22:53 +0800 Subject: [PATCH 28/55] refactor(sync): update election timer range --- source/libs/sync/inc/syncEnv.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 4f91990ccf..2b2cb7efae 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -28,13 +28,13 @@ extern "C" { #include "trpc.h" #include "ttimer.h" -#define TIMER_MAX_MS 0x7FFFFFFF -#define ENV_TICK_TIMER_MS 1000 -#define PING_TIMER_MS 1000 -#define ELECT_TIMER_MS_MIN 500 -#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) +#define TIMER_MAX_MS 0x7FFFFFFF +#define ENV_TICK_TIMER_MS 1000 +#define PING_TIMER_MS 1000 +#define ELECT_TIMER_MS_MIN 1300 +#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 100 +#define HEARTBEAT_TIMER_MS 1000 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) From cdb6b2429a92c254757f6c9a30dd45373e8fa009 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 23 Jun 2022 14:37:29 +0800 Subject: [PATCH 29/55] refactor(sync): update election timer range --- source/libs/sync/inc/syncEnv.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 2b2cb7efae..2e8e0e1227 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -34,7 +34,7 @@ extern "C" { #define ELECT_TIMER_MS_MIN 1300 #define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 1000 +#define HEARTBEAT_TIMER_MS 900 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) From 00c0926c44739074cf54f3e913ccf58f54868bc7 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 23 Jun 2022 14:58:52 +0800 Subject: [PATCH 30/55] refactor(sync): hold wal handle with log store --- source/libs/sync/inc/syncRaftLog.h | 8 +++--- source/libs/sync/src/syncMain.c | 8 ++++-- source/libs/sync/src/syncRaftCfg.c | 38 +++++++++++++++------------- source/libs/sync/src/syncRaftLog.c | 40 +++++++++++++++++++++--------- 4 files changed, 60 insertions(+), 34 deletions(-) diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index aec1f77b42..bc848d217c 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -26,11 +26,13 @@ extern "C" { #include "syncInt.h" #include "syncRaftEntry.h" #include "taosdef.h" +#include "wal.h" typedef struct SSyncLogStoreData { - SSyncNode* pSyncNode; - SWal* pWal; - SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 + SSyncNode* pSyncNode; + SWal* pWal; + SWalReadHandle* pWalHandle; + SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 } SSyncLogStoreData; SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 8979f244f0..9a1b5e687e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1311,6 +1311,10 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg)); + char* printStr = ""; + if (pCfgStr != NULL) { + printStr = pCfgStr; + } if (userStrLen < 256) { char logBuf[256 + 256]; @@ -1322,7 +1326,7 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, - pSyncNode->changing, pCfgStr); + pSyncNode->changing, printStr); } else { snprintf(logBuf, sizeof(logBuf), "%s", str); } @@ -1339,7 +1343,7 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, - pSyncNode->changing, pCfgStr); + pSyncNode->changing, printStr); } else { snprintf(s, len, "%s", str); } diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 2cc1eb0239..f40eff57d9 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -107,26 +107,30 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) { } char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { - int32_t len = 512; - char *s = taosMemoryMalloc(len); - memset(s, 0, len); + if (pSyncCfg != NULL) { + int32_t len = 512; + char *s = taosMemoryMalloc(len); + memset(s, 0, len); - snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); - char *p = s + strlen(s); - for (int i = 0; i < pSyncCfg->replicaNum; ++i) { - /* - if (p + 128 + 32 > s + len) { - break; + snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); + char *p = s + strlen(s); + for (int i = 0; i < pSyncCfg->replicaNum; ++i) { + /* + if (p + 128 + 32 > s + len) { + break; + } + */ + char buf[128 + 32]; + snprintf(buf, sizeof(buf), "%s:%d, ", pSyncCfg->nodeInfo[i].nodeFqdn, pSyncCfg->nodeInfo[i].nodePort); + strncpy(p, buf, sizeof(buf)); + p = s + strlen(s); } - */ - char buf[128 + 32]; - snprintf(buf, sizeof(buf), "%s:%d, ", pSyncCfg->nodeInfo[i].nodeFqdn, pSyncCfg->nodeInfo[i].nodePort); - strncpy(p, buf, sizeof(buf)); - p = s + strlen(s); - } - strcpy(p - 2, "}"); + strcpy(p - 2, "}"); - return s; + return s; + } + + return NULL; } int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) { diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index f000c9f117..89fd613307 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -16,7 +16,6 @@ #include "syncRaftLog.h" #include "syncRaftCfg.h" #include "syncRaftStore.h" -#include "wal.h" // refactor, log[0 .. n] ==> log[m .. n] static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); @@ -233,7 +232,8 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, *ppEntry = NULL; - SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + SWalReadHandle* pWalHandle = pData->pWalHandle; if (pWalHandle == NULL) { return -1; } @@ -256,9 +256,11 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, } } while (0); - int32_t saveErr = terrno; - walCloseReadHandle(pWalHandle); - terrno = saveErr; + /* + int32_t saveErr = terrno; + walCloseReadHandle(pWalHandle); + terrno = saveErr; + */ return code; } @@ -274,9 +276,11 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, ASSERT((*ppEntry)->dataLen == pWalHandle->pHead->head.bodyLen); memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen); - int32_t saveErr = terrno; - walCloseReadHandle(pWalHandle); - terrno = saveErr; + /* + int32_t saveErr = terrno; + walCloseReadHandle(pWalHandle); + terrno = saveErr; + */ return code; } @@ -319,6 +323,10 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { SSyncLogStoreData* pData = pLogStore->data; pData->pSyncNode = pSyncNode; pData->pWal = pSyncNode->pWal; + ASSERT(pData->pWal != NULL); + + pData->pWalHandle = walOpenReadHandle(pData->pWal); + ASSERT(pData->pWalHandle != NULL); SyncIndex firstVer = walGetFirstVer(pData->pWal); SyncIndex lastVer = walGetLastVer(pData->pWal); @@ -357,6 +365,11 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { void logStoreDestory(SSyncLogStore* pLogStore) { if (pLogStore != NULL) { + SSyncLogStoreData* pData = pLogStore->data; + if (pData->pWalHandle != NULL) { + walCloseReadHandle(pData->pWalHandle); + } + taosMemoryFree(pLogStore->data); taosMemoryFree(pLogStore); } @@ -405,7 +418,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { SWal* pWal = pData->pWal; if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { - SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + SWalReadHandle* pWalHandle = pData->pWalHandle; ASSERT(pWalHandle != NULL); int32_t code = walReadWithHandle(pWalHandle, index); @@ -441,9 +455,11 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { ASSERT(pEntry->dataLen == pWalHandle->pHead->head.bodyLen); memcpy(pEntry->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen); - int32_t saveErr = terrno; - walCloseReadHandle(pWalHandle); - terrno = saveErr; + /* + int32_t saveErr = terrno; + walCloseReadHandle(pWalHandle); + terrno = saveErr; + */ return pEntry; From 147a984c17631d29392bb346d4670d7db6ad6d7c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 15:11:23 +0800 Subject: [PATCH 31/55] feat: make config dnode work --- source/common/src/tglobal.c | 11 ++++++++--- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 11 +++++++++-- source/util/src/tlog.c | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 6cae3a13e6..c05d4cf1be 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -760,9 +760,14 @@ void taosCleanupCfg() { } void taosCfgDynamicOptions(const char *option, const char *value) { - if (strcasecmp(option, "debugFlag") == 0) { - int32_t debugFlag = atoi(value); - taosSetAllDebugFlag(debugFlag); + if (strncasecmp(option, "debugFlag", 9) == 0) { + if (value != NULL) { + if (strlen(option) > 10) { + value = option + 10; + } + } + int32_t flag = atoi(value); + taosSetAllDebugFlag(flag); } if (strcasecmp(option, "resetlog") == 0) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 9c8918a445..44b6f0ee85 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -123,8 +123,15 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { - dError("config req is received, but not supported yet"); - return TSDB_CODE_OPS_NOT_SUPPORT; + SDCfgDnodeReq cfgReq = {0}; + if (tDeserializeSMCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); + taosCfgDynamicOptions(cfgReq.config, cfgReq.value); + return 0; } static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 353e94a490..0439ed148b 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -743,7 +743,7 @@ cmp_end: } void taosSetAllDebugFlag(int32_t flag) { - if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return; + if (flag <= 0) return; dDebugFlag = flag; vDebugFlag = flag; From 983bf5b865a6db2594dd6cbebc7ffc1b8becad7e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 23 Jun 2022 15:26:42 +0800 Subject: [PATCH 32/55] fix(wal): read handle version --- source/libs/wal/src/walRead.c | 15 +++++++++++++-- source/libs/wal/src/walWrite.c | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 9bed6d230b..682afbb785 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -272,14 +272,23 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } } + if (ver > pRead->pWal->vers.lastVer || ver < pRead->pWal->vers.firstVer) { + wError("invalid version: % " PRId64 ", first ver %ld, last ver %ld", ver, pRead->pWal->vers.firstVer, + pRead->pWal->vers.lastVer); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; + } + ASSERT(taosValidFile(pRead->pReadLogTFile) == true); code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { if (code < 0) terrno = TAOS_SYSTEM_ERROR(errno); - else + else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); + } return -1; } @@ -304,8 +313,10 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { pRead->pHead->head.bodyLen) { if (code < 0) terrno = TAOS_SYSTEM_ERROR(errno); - else + else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); + } return -1; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 031289f93e..34f83fb8a9 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -150,6 +150,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { ASSERT(code == 0); if (code != 0) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); return -1; } if (head.head.version != ver) { From cbacd1fd4578570836657a8332478dfad5f2e0a0 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 15:35:19 +0800 Subject: [PATCH 33/55] fix: fix empty downstream input for group sort operator --- source/libs/executor/src/sortoperator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 6086138301..24016c3338 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -387,6 +387,10 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { pInfo->hasGroupId = true; pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]); + if (pInfo->prefetchedSortInput == NULL) { + doSetOperatorCompleted(pOperator); + return NULL; + } pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId; pInfo->childOpStatus = CHILD_OP_NEW_GROUP; beginSortGroup(pOperator); From 0811e5fc93416e883632a9df2ac30ba3dc6ce054 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 15:35:22 +0800 Subject: [PATCH 34/55] feat: make config dnode work --- include/common/tmsg.h | 10 ++++- source/common/src/tglobal.c | 5 --- source/common/src/tmsg.c | 27 +++++++++++++ source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 43 +++++++++++++++------ 5 files changed, 69 insertions(+), 18 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0ef9dffa2e..c32b7be941 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1346,11 +1346,19 @@ typedef struct { int32_t dnodeId; char config[TSDB_DNODE_CONFIG_LEN]; char value[TSDB_DNODE_VALUE_LEN]; -} SMCfgDnodeReq, SDCfgDnodeReq; +} SMCfgDnodeReq; int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); +typedef struct { + char config[TSDB_DNODE_CONFIG_LEN]; + char value[TSDB_DNODE_VALUE_LEN]; +} SDCfgDnodeReq; + +int32_t tSerializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq); +int32_t tDeserializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq); + typedef struct { int32_t dnodeId; } SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c05d4cf1be..2172a6fe50 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -761,11 +761,6 @@ void taosCleanupCfg() { void taosCfgDynamicOptions(const char *option, const char *value) { if (strncasecmp(option, "debugFlag", 9) == 0) { - if (value != NULL) { - if (strlen(option) > 10) { - value = option + 10; - } - } int32_t flag = atoi(value); taosSetAllDebugFlag(flag); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6bd0dc02e1..4ad11f2998 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1500,6 +1500,33 @@ int32_t tDeserializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq return 0; } +int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->config) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->value) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->config) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->value) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 44b6f0ee85..1ffd86beb1 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -124,7 +124,7 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { SDCfgDnodeReq cfgReq = {0}; - if (tDeserializeSMCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { + if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index b159061ce3..d3723326b9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -502,20 +502,20 @@ _OVER: } static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - SDnodeObj *pObj = NULL; - void *pIter = NULL; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + SDnodeObj *pObj = NULL; + void *pIter = NULL; SDnodeListRsp rsp = {0}; - int32_t code = -1; - + int32_t code = -1; + rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet)); if (NULL == rsp.dnodeList) { mError("failed to alloc epSet while process dnode list req"); terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - + while (1) { pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj); if (pIter == NULL) break; @@ -554,7 +554,6 @@ _OVER: return code; } - static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -732,15 +731,37 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr()); return -1; } - SEpSet epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - int32_t bufLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); + SDCfgDnodeReq dcfgReq = {0}; + if (strncasecmp(cfgReq.config, "debugFlag", 9) == 0) { + const char *value = cfgReq.value; + int32_t flag = atoi(value); + if (flag <= 0) { + flag = atoi(cfgReq.config + 10); + } + if (flag <= 0 || flag > 255) { + mError("dnode:%d, failed to config debugFlag since value:%d", cfgReq.dnodeId, flag); + terrno = TSDB_CODE_INVALID_CFG; + return -1; + } + + strcpy(dcfgReq.config, "debugFlag"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); + } else if (strcasecmp(cfgReq.config, "resetlog") == 0) { + strcpy(dcfgReq.config, "resetlog"); + } else { + terrno = TSDB_CODE_INVALID_CFG; + mError("dnode:%d, failed to config since %s", cfgReq.dnodeId, terrstr()); + return -1; + } + + int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, &dcfgReq); void *pBuf = rpcMallocCont(bufLen); if (pBuf == NULL) return -1; - tSerializeSMCfgDnodeReq(pBuf, bufLen, &cfgReq); + tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq); mDebug("dnode:%d, send config req to dnode, app:%p", cfgReq.dnodeId, pReq->info.ahandle); SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; From 4fdc62a27fc77bd94c8bc13eb1b039e0f8de7d8e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 23 Jun 2022 16:32:06 +0800 Subject: [PATCH 35/55] refactor(sync): delete fake match branch --- source/libs/sync/src/syncAppendEntries.c | 23 +++++++++++++++++++---- source/libs/sync/src/syncReplication.c | 9 +++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 31176d942e..4df446224c 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -502,6 +502,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs } } while (0); +#if 0 // fake match // // condition1: @@ -535,7 +536,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = condition1 || condition2 || condition3 || condition4; if (condition) { - syncNodeEventLog(ths, "recv sync-append-entries, fake match"); + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, fake match, pre-index:%ld, pre-term:%lu", + pMsg->prevLogIndex, pMsg->prevLogTerm); + syncNodeEventLog(ths, logBuf); // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); @@ -555,6 +559,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs return ret; } } while (0); +#endif // fake match2 // @@ -569,7 +574,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && (pMsg->prevLogIndex <= ths->commitIndex); if (condition) { - syncNodeEventLog(ths, "recv sync-append-entries, fake match2"); + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, fake match2, pre-index:%ld, pre-term:%lu", + pMsg->prevLogIndex, pMsg->prevLogTerm); + syncNodeEventLog(ths, logBuf); SyncIndex matchIndex = ths->commitIndex; bool hasAppendEntries = pMsg->dataLen > 0; @@ -597,6 +605,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = syncNodePreCommit(ths, pAppendEntry); ASSERT(code == 0); + // update match index matchIndex = pMsg->prevLogIndex + 1; syncEntryDestory(pAppendEntry); @@ -642,7 +651,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = condition1 || condition2; if (condition) { - syncNodeEventLog(ths, "recv sync-append-entries, not match"); + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, not match, pre-index:%ld, pre-term:%lu", + pMsg->prevLogIndex, pMsg->prevLogTerm); + syncNodeEventLog(ths, logBuf); // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); @@ -681,7 +693,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs // has entries in SyncAppendEntries msg bool hasAppendEntries = pMsg->dataLen > 0; - syncNodeEventLog(ths, "recv sync-append-entries, match"); + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu", + pMsg->prevLogIndex, pMsg->prevLogTerm); + syncNodeEventLog(ths, logBuf); if (hasExtraEntries) { // make log same, rollback deleted entries diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index c378926e28..68f71e55b9 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -219,6 +219,15 @@ int32_t syncNodeReplicate(SSyncNode* pSyncNode) { int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg) { int32_t ret = 0; + do { + char host[128]; + uint16_t port; + syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port); + sDebug("vgId:%d, send append-entries to %s:%d, term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, commit:%ld", + pSyncNode->vgId, host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, + pMsg->commitIndex); + } while (0); + SRpcMsg rpcMsg; syncAppendEntries2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); From d6ccf5720dcae58ca63dc206259efa4e59339fbc Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 23 Jun 2022 15:43:39 +0800 Subject: [PATCH 36/55] fix(stream): valgrind memory error --- source/libs/executor/src/timewindowoperator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 19f0fd4ea7..6813c63542 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2238,7 +2238,8 @@ static void clearUpdateDataBlock(SSDataBlock* pBlock) { } void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) { - ASSERT(pDest->info.capacity >= pSource->info.rows); + // ASSERT(pDest->info.capacity >= pSource->info.rows); + blockDataEnsureCapacity(pDest, pSource->info.rows); clearUpdateDataBlock(pDest); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex); From e07164961973dc52279ae0c676c23769ffe75fdc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 16:34:43 +0800 Subject: [PATCH 37/55] test: update the unit test. --- include/common/tcommon.h | 2 +- source/common/src/tdatablock.c | 3 +++ .../libs/scalar/test/scalar/scalarTests.cpp | 25 ++++++++++++------- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index e8fddee869..4a113f5822 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -68,7 +68,7 @@ typedef struct SColumnDataAgg { typedef struct SDataBlockInfo { STimeWindow window; - int32_t rows; + int32_t rows; // todo hide this attribute int32_t rowSize; uint64_t uid; // the uid of table, from which current data block comes uint16_t blockId; // block id, generated by physical planner diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 15425c8895..8317c6c79d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1109,6 +1109,9 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo * return TSDB_CODE_SUCCESS; } + // todo temp disable it +// ASSERT(pColumn->info.bytes != 0); + int32_t existedRows = pBlockInfo->rows; if (IS_VAR_DATA_TYPE(pColumn->info.type)) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 3324981531..e49c11c390 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -86,15 +86,17 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s if (newBlock) { SSDataBlock *res = createDataBlock(); - SColumnInfoData idata = {0}; - idata.info = *colInfo; + SColumnInfoData idata = {.info = *colInfo}; + colInfoDataEnsureCapacity(&idata, rows); + blockDataAppendColInfo(res, &idata); - blockDataEnsureCapacity(res, rows); + res->info.capacity = rows; res->info.rows = rows; + SColumnInfoData* p = static_cast(taosArrayGet(res->pDataBlock, 0)); + ASSERT(p->pData != NULL && p->nullbitmap != NULL); taosArrayPush(pBlockList, &res); - *dataBlockId = taosArrayGetSize(pBlockList) - 1; res->info.blockId = *dataBlockId; *slotId = 0; @@ -102,6 +104,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); SColumnInfoData idata = {0}; idata.info = *colInfo; + colInfoDataEnsureCapacity(&idata, rows); blockDataAppendColInfo(res, &idata); *dataBlockId = taosArrayGetSize(pBlockList) - 1; @@ -140,17 +143,18 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in if (NULL == *block) { SSDataBlock *res = createDataBlock(); - res->info.rows = rowNum; for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, i + 1); - int32_t size = idata.info.bytes * rowNum; + SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_INT, 10, i + 1); + colInfoDataEnsureCapacity(&idata, rowNum); blockDataAppendColInfo(res, &idata); } SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3); + colInfoDataEnsureCapacity(&idata, rowNum); blockDataAppendColInfo(res, &idata); - blockDataEnsureCapacity(res, rowNum); + res->info.capacity = rowNum; + res->info.rows = rowNum; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -170,8 +174,10 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in int32_t idx = taosArrayGetSize(res->pDataBlock); SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1 + idx); + colInfoDataEnsureCapacity(&idata, rowNum); + + res->info.capacity = rowNum; blockDataAppendColInfo(res, &idata); - blockDataEnsureCapacity(res, rowNum); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); @@ -1507,6 +1513,7 @@ TEST(columnTest, bigint_column_multi_binary_column) { SArray *blockList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(blockList, &src); + SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); int16_t dataBlockId = 0, slotId = 0; scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); From ef00464f9c1b8c9ba7fd4e5b156c384d5cc6c308 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 16:35:05 +0800 Subject: [PATCH 38/55] fix: retry 12 seconds if lock file failed --- source/dnode/mgmt/node_util/src/dmFile.c | 16 +++++++++++++--- tests/script/tsim/mnode/basic5.sim | 1 - 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 78e706f908..2185adc18b 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -127,14 +127,24 @@ TdFilePtr dmCheckRunning(const char *dataDir) { return NULL; } - int32_t ret = taosLockFile(pFile); - if (ret != 0) { + int32_t retryTimes = 0; + int32_t ret = 0; + do { + ret = taosLockFile(pFile); + if (ret == 0) break; + terrno = TAOS_SYSTEM_ERROR(errno); + taosMsleep(100); + retryTimes++; + dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes); + } while (retryTimes < 120); + + if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s", filepath, terrstr()); taosCloseFile(&pFile); return NULL; } + terrno = 0; dDebug("lock file:%s to prevent repeated starts", filepath); return pFile; } diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index e298dcbe55..23f5f6d782 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -233,7 +233,6 @@ if $data(1)[3] != dropping then endi print =============== step9: start mnode1 and wait it dropped -sleep 3000 system sh/exec.sh -n dnode1 -s start $x = 0 From b556d1fa62746d95c988a8dcc8804922d76d3df7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 16:37:36 +0800 Subject: [PATCH 39/55] fix(query): fix to_ISO8601 function crash when processing constant illegal inputs TD-15211 --- source/libs/function/src/builtins.c | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6baf45a5b3..f6d2fdadae 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1267,6 +1267,29 @@ static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) { return false; } +static bool validateTimestampDigits(const SValueNode* pVal) { + if (IS_INTEGER_TYPE(pVal->node.resType.type)) { + return false; + } + + int64_t tsVal = pVal->datum.i; + char fraction[20] = {0}; + NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction); + int32_t tsDigits = (int32_t)strlen(fraction); + + if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) { + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || + tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || + tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + return true; + } else { + return false; + } + } + + return true; +} + static bool validateTimezoneFormat(const SValueNode* pVal) { if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) { return false; @@ -1379,6 +1402,15 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0))) { + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); + + if (!validateTimestampDigits(pValue)) { + pFunc->node.resType = (SDataType){.bytes = 0, .type = TSDB_DATA_TYPE_BINARY}; + return TSDB_CODE_SUCCESS; + } + } + // param1 if (numOfParams == 2) { SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); From c449b7208acdec8656f03af90fce064192b57b9d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 16:52:06 +0800 Subject: [PATCH 40/55] fix: set correct operator blocking and add closing rpc trace to udfd --- source/libs/executor/src/sortoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 2 ++ source/libs/function/src/udfd.c | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 4b5ad7b123..81cf8d6527 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -460,7 +460,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pInfo->pColMatchInfo = pColMatchColInfo; pOperator->name = "GroupSortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT; - pOperator->blocking = true; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->exprSupp.pExprInfo = pExprInfo; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 19f0fd4ea7..7725a939d6 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4117,6 +4117,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI goto _error; } + miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; iaInfo->win = pTaskInfo->window; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 7fffa84e0b..f8713e4b47 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -613,7 +613,9 @@ int32_t udfdOpenClientRpc() { } int32_t udfdCloseClientRpc() { + fnInfo("udfd begin closing rpc"); rpcClose(global.clientRpc); + fnInfo("udfd finish closing rpc"); return 0; } @@ -937,7 +939,6 @@ int main(int argc, char *argv[]) { uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); udfdRun(); - removeListeningPipe(); uv_thread_join(&mnodeConnectThread); udfdCloseClientRpc(); From f140a5ad1b4d5fd0b68ff2c5b4d568eb87ca753c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 23 Jun 2022 16:57:18 +0800 Subject: [PATCH 41/55] refactor(sync): add trace log --- source/libs/sync/src/syncAppendEntries.c | 12 ++++++------ source/libs/sync/src/syncReplication.c | 8 +++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 4df446224c..073fdd104d 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -575,8 +575,8 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs (pMsg->prevLogIndex <= ths->commitIndex); if (condition) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, fake match2, pre-index:%ld, pre-term:%lu", - pMsg->prevLogIndex, pMsg->prevLogTerm); + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); syncNodeEventLog(ths, logBuf); SyncIndex matchIndex = ths->commitIndex; @@ -652,8 +652,8 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs if (condition) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, not match, pre-index:%ld, pre-term:%lu", - pMsg->prevLogIndex, pMsg->prevLogTerm); + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, not match, pre-index:%ld, pre-term:%lu, datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); syncNodeEventLog(ths, logBuf); // prepare response msg @@ -694,8 +694,8 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool hasAppendEntries = pMsg->dataLen > 0; char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu", - pMsg->prevLogIndex, pMsg->prevLogTerm); + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu, datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); syncNodeEventLog(ths, logBuf); if (hasExtraEntries) { diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 68f71e55b9..02cd977e1e 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -223,9 +223,11 @@ int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, c char host[128]; uint16_t port; syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port); - sDebug("vgId:%d, send append-entries to %s:%d, term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, commit:%ld", - pSyncNode->vgId, host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, - pMsg->commitIndex); + sDebug( + "vgId:%d, send sync-append-entries to %s:%d, term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, commit:%ld, " + "datalen:%d", + pSyncNode->vgId, host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, + pMsg->commitIndex, pMsg->dataLen); } while (0); SRpcMsg rpcMsg; From 323ee6b1b00861a41d6af73746112110e1341257 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 23 Jun 2022 17:26:23 +0800 Subject: [PATCH 42/55] enh(tmq): add demo for tmq_get_raw_meta --- examples/c/tmq.c | 6 +++++- include/client/taos.h | 2 +- source/client/src/tmq.c | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 25e4b5c193..6fb7e7a1fc 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -27,7 +27,11 @@ static void msg_process(TAOS_RES* msg) { printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { - printf("meta, skip\n"); + void* meta; + int32_t metaLen; + tmq_get_raw_meta(msg, &meta, &metaLen); + + printf("meta, len is %d\n", metaLen); return; } while (1) { diff --git a/include/client/taos.h b/include/client/taos.h index 9d4da221f4..d31d5c582c 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -261,7 +261,7 @@ enum tmq_res_t { typedef enum tmq_res_t tmq_res_t; DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); -DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, const void **raw_meta, int32_t *raw_meta_len); +DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, void **raw_meta, int32_t *raw_meta_len); DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 6892a4d1a7..637a7ee5dd 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1871,7 +1871,7 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -int32_t tmq_get_raw_meta(TAOS_RES* res, const void** raw_meta, int32_t* raw_meta_len) { +int32_t tmq_get_raw_meta(TAOS_RES* res, void** raw_meta, int32_t* raw_meta_len) { if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; *raw_meta = pMetaRspObj->metaRsp.metaRsp; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9eec1558b2..cca3a58588 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -306,7 +306,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE || pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE || pHead->msgType == TDMT_VND_DROP_TTL_TABLE); - // return + tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType); SMqMetaRsp metaRsp = {0}; metaRsp.reqOffset = pReq->currentOffset; metaRsp.rspOffset = fetchOffset; From 7fda5a861e58595fb0ad07f5a00987c7016a7dbb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 16:37:36 +0800 Subject: [PATCH 43/55] fix(query): fix to_ISO8601 function crash when processing constant illegal inputs TD-15211 --- source/libs/function/src/builtins.c | 2 +- source/libs/scalar/src/sclfunc.c | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index f6d2fdadae..f985cc324d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1268,7 +1268,7 @@ static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) { } static bool validateTimestampDigits(const SValueNode* pVal) { - if (IS_INTEGER_TYPE(pVal->node.resType.type)) { + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { return false; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index d335b04ea4..a5154fac39 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -856,10 +856,13 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); + bool tzPresent = (inputNum == 2) ? true : false; char* tz; int32_t tzLen; - tz = varDataVal(pInput[1].columnData->pData); - tzLen = varDataLen(pInput[1].columnData->pData); + if (tzPresent) { + tz = varDataVal(pInput[1].columnData->pData); + tzLen = varDataLen(pInput[1].columnData->pData); + } for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { From 58836d2ddf9e0b03f480759f10dc2cd6d60967eb Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 17:35:55 +0800 Subject: [PATCH 44/55] feat: refactor rpc quit --- source/libs/transport/src/transCli.c | 38 +++++++++++++++------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index bf03ea26ba..0dd00e0dc2 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -983,17 +983,18 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (pCtx->retryCount < pEpSet->numOfEps * 3) { pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + if (pThrd->quit == false) { + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + transPrintEpSet(pEpSet); + tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, pEpSet->numOfEps * 3); - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - transPrintEpSet(pEpSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, pEpSet->numOfEps * 3); - - transUnrefCliHandle(pConn); - return -1; + transUnrefCliHandle(pConn); + return -1; + } } } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { if (pResp->contLen == 0) { @@ -1010,15 +1011,16 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); } - if (pConn->status != ConnInPool) { - addConnToPool(pThrd->pool, pConn); + if (pThrd->quit == false) { + if (pConn->status != ConnInPool) { + addConnToPool(pThrd->pool, pConn); + } + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + return -1; } - - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - return -1; } } From 9c6057caa38c640887597bb690d20548e8761de2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 17:44:58 +0800 Subject: [PATCH 45/55] fix(query): fix syntax error. --- source/dnode/vnode/src/tq/tqExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 2d9cfbc0a1..da052ee3eb 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -75,7 +75,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp if (pDataBlock == NULL) break; ASSERT(pDataBlock->info.rows != 0); - ASSERT(pDataBlock->info.numOfCols != 0); + ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0); tqAddBlockDataToRsp(pDataBlock, pRsp); pRsp->blockNum++; From 9278de28b41c57a1796f617958f712b867ba615c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 17:45:16 +0800 Subject: [PATCH 46/55] fix: improve udfd exist speed --- source/libs/function/src/udfd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index f8713e4b47..838071dbf1 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -939,8 +939,8 @@ int main(int argc, char *argv[]) { uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); udfdRun(); + removeListeningPipe(); - uv_thread_join(&mnodeConnectThread); udfdCloseClientRpc(); return 0; From d01e4031bd31d4c81a216ef022390f3c7d7f5098 Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 23 Jun 2022 18:01:49 +0800 Subject: [PATCH 47/55] test: modify testcase of muti-mnodes --- tests/pytest/util/cluster.py | 264 ++---------------- .../system-test/6-cluster/5dnode3mnodeStop.py | 10 - tests/system-test/fulltest.sh | 2 +- tests/system-test/test.py | 18 +- 4 files changed, 40 insertions(+), 254 deletions(-) diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py index 5debfab2c4..6d3be8c7a9 100644 --- a/tests/pytest/util/cluster.py +++ b/tests/pytest/util/cluster.py @@ -21,45 +21,41 @@ class ClusterDnodes(TDDnodes): self.testCluster = False self.valgrind = 0 self.killValgrind = 1 - self.independent = True - self.dnodeNums = 5 - - # def getTDDnodes(dnodeNums): - - # return class ConfigureyCluster: - """configure dnodes and return TDDnodes list, it can """ + """This will create defined number of dnodes and create a cluset. + at the same time, it will return TDDnodes list: dnodes, """ + hostname= socket.gethostname() def __init__(self): - self.dnodes = None - self.dnodes_nums = 5 + self.dnodes = [] + self.dnodeNums = 5 self.independent = True - self.start_port = 6030 + self.startPort = 6030 self.portStep = 100 - hostname1= socket.gethostname() + self.mnodeNums = 0 - def configure_cluster(self ,dnodes_nums=5,independent=True,start_port=6030,portStep=100,hostname="%s"%hostname1): - self.start_port=int(start_port) + def configure_cluster(self ,dnodeNums=5,mnodeNums=0,startPort=6030,portStep=100,hostname="%s"%hostname): + self.startPort=int(startPort) self.portStep=int(portStep) self.hostname=hostname - self.dnodes_nums = int(dnodes_nums) - self.independent = independent + self.dnodeNums = int(dnodeNums) + self.mnodeNums = int(mnodeNums) self.dnodes = [] - start_port_sec = 6130 - for num in range(1, (self.dnodes_nums+1)): + startPort_sec = int(startPort+portStep) + for num in range(1, (self.dnodeNums+1)): dnode = TDDnode(num) - dnode.addExtraCfg("firstEp", f"{hostname}:{self.start_port}") + dnode.addExtraCfg("firstEp", f"{hostname}:{self.startPort}") dnode.addExtraCfg("fqdn", f"{hostname}") - dnode.addExtraCfg("serverPort", f"{self.start_port + (num-1)*self.portStep}") - # dnode.addExtraCfg("monitorFqdn", hostname) - # dnode.addExtraCfg("monitorPort", 7043) - dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}") - # configure three dnoe don't support vnodes - if self.dnodes_nums > 4 : - if self.independent and (num < 4): - dnode.addExtraCfg("supportVnodes", 0) + dnode.addExtraCfg("serverPort", f"{self.startPort + (num-1)*self.portStep}") + dnode.addExtraCfg("secondEp", f"{hostname}:{startPort_sec}") + print(num,self.mnodeNums) + + # configure dnoe of independent mnodes + if num <= self.mnodeNums and self.mnodeNums != 0 : + dnode.addExtraCfg("supportVnodes", 0) + print("-----------------88888123131231320999999") # print(dnode) self.dnodes.append(dnode) return self.dnodes @@ -69,24 +65,7 @@ class ConfigureyCluster: for dnode in self.dnodes[1:]: # print(dnode.cfgDict) dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] - print(dnode_id) tdSql.execute(" create dnode '%s';"%dnode_id) - # count=0 - # while count < 10: - # time.sleep(1) - # tdSql.query("show dnodes;") - # if tdSql.checkRows(self.dnodes_nums) : - # print("mnode is three nodes") - # if tdSql.queryResult[0][4]=='leader' : - # if tdSql.queryResult[2][4]=='offline': - # if tdSql.queryResult[1][2]=='follower': - # print("stop mnodes on dnode 3 successfully in 10s") - # break - # count+=1 - # else: - # print("stop mnodes on dnode 3 failed in 10s") - # return -1 - checkstatus=False @@ -97,206 +76,19 @@ class ConfigureyCluster: tdSql.query("show dnodes") # tdLog.debug(tdSql.queryResult) status=0 - for i in range(self.dnodes_nums): + for i in range(self.dnodeNums): if tdSql.queryResult[i][4] == "ready": status+=1 - tdLog.debug(status) + # tdLog.debug(status) - if status == self.dnodes_nums: - tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodes_nums) + if status == self.dnodeNums: + tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodeNums) break count+=1 time.sleep(1) else: - tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodes_nums) + tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodeNums) return -1 -cluster = ConfigureyCluster() - - # def start(self ,dnodes_nums): - - # self.TDDnodes = ClusterDnodes(dnodes) - # self.TDDnodes.init("") - # self.TDDnodes.setTestCluster(testCluster) - # self.TDDnodes.setValgrind(valgrind) - # self.TDDnodes.stopAll() - # for dnode in self.TDDnodes.dnodes: - # self.TDDnodes.deploy(dnode.index,{}) - - # for dnode in self.TDDnodes.dnodes: - # self.TDDnodes.starttaosd(dnode.index) - - # # create cluster - # for dnode in self.TDDnodes.dnodes[1:]: - # # print(dnode.cfgDict) - # dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] - # dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] - # dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] - # cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" - # print(cmd) - # os.system(cmd) - - # time.sleep(2) - # tdLog.info(" create cluster with %d dnode done! " %dnodes_nums) - - # def buildcluster(self,dnodenumber): - # self.depoly_cluster(dnodenumber) - # self.master_dnode = self.TDDnodes.dnodes[0] - # self.host=self.master_dnode.cfgDict["fqdn"] - # conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir) - # tdSql.init(conn1.cursor()) - - # def checkdnodes(self,dnodenumber): - # count=0 - # while count < 10: - # time.sleep(1) - # statusReadyBumber=0 - # tdSql.query("show dnodes;") - # if tdSql.checkRows(dnodenumber) : - # print("dnode is %d nodes"%dnodenumber) - # for i in range(dnodenumber): - # if tdSql.queryResult[i][4] !='ready' : - # status=tdSql.queryResult[i][4] - # print("dnode:%d status is %s "%(i,status)) - # break - # else: - # statusReadyBumber+=1 - # print(statusReadyBumber) - # if statusReadyBumber == dnodenumber : - # print("all of %d mnodes is ready in 10s "%dnodenumber) - # return True - # break - # count+=1 - # else: - # print("%d mnodes is not ready in 10s "%dnodenumber) - # return False - - - # def check3mnode(self): - # count=0 - # while count < 10: - # time.sleep(1) - # tdSql.query("show mnodes;") - # if tdSql.checkRows(3) : - # print("mnode is three nodes") - # if tdSql.queryResult[0][2]=='leader' : - # if tdSql.queryResult[1][2]=='follower': - # if tdSql.queryResult[2][2]=='follower': - # print("three mnodes is ready in 10s") - # break - # elif tdSql.queryResult[0][2]=='follower' : - # if tdSql.queryResult[1][2]=='leader': - # if tdSql.queryResult[2][2]=='follower': - # print("three mnodes is ready in 10s") - # break - # elif tdSql.queryResult[0][2]=='follower' : - # if tdSql.queryResult[1][2]=='follower': - # if tdSql.queryResult[2][2]=='leader': - # print("three mnodes is ready in 10s") - # break - # count+=1 - # else: - # print("three mnodes is not ready in 10s ") - # return -1 - - # tdSql.query("show mnodes;") - # tdSql.checkRows(3) - # tdSql.checkData(0,1,'%s:6030'%self.host) - # tdSql.checkData(0,3,'ready') - # tdSql.checkData(1,1,'%s:6130'%self.host) - # tdSql.checkData(1,3,'ready') - # tdSql.checkData(2,1,'%s:6230'%self.host) - # tdSql.checkData(2,3,'ready') - - # def check3mnode1off(self): - # count=0 - # while count < 10: - # time.sleep(1) - # tdSql.query("show mnodes;") - # if tdSql.checkRows(3) : - # print("mnode is three nodes") - # if tdSql.queryResult[0][2]=='offline' : - # if tdSql.queryResult[1][2]=='leader': - # if tdSql.queryResult[2][2]=='follower': - # print("stop mnodes on dnode 2 successfully in 10s") - # break - # elif tdSql.queryResult[1][2]=='follower': - # if tdSql.queryResult[2][2]=='leader': - # print("stop mnodes on dnode 2 successfully in 10s") - # break - # count+=1 - # else: - # print("stop mnodes on dnode 2 failed in 10s ") - # return -1 - # tdSql.error("drop mnode on dnode 1;") - - # tdSql.query("show mnodes;") - # tdSql.checkRows(3) - # tdSql.checkData(0,1,'%s:6030'%self.host) - # tdSql.checkData(0,2,'offline') - # tdSql.checkData(0,3,'ready') - # tdSql.checkData(1,1,'%s:6130'%self.host) - # tdSql.checkData(1,3,'ready') - # tdSql.checkData(2,1,'%s:6230'%self.host) - # tdSql.checkData(2,3,'ready') - - # def check3mnode2off(self): - # count=0 - # while count < 40: - # time.sleep(1) - # tdSql.query("show mnodes;") - # if tdSql.checkRows(3) : - # print("mnode is three nodes") - # if tdSql.queryResult[0][2]=='leader' : - # if tdSql.queryResult[1][2]=='offline': - # if tdSql.queryResult[2][2]=='follower': - # print("stop mnodes on dnode 2 successfully in 10s") - # break - # count+=1 - # else: - # print("stop mnodes on dnode 2 failed in 10s ") - # return -1 - # tdSql.error("drop mnode on dnode 2;") - - # tdSql.query("show mnodes;") - # tdSql.checkRows(3) - # tdSql.checkData(0,1,'%s:6030'%self.host) - # tdSql.checkData(0,2,'leader') - # tdSql.checkData(0,3,'ready') - # tdSql.checkData(1,1,'%s:6130'%self.host) - # tdSql.checkData(1,2,'offline') - # tdSql.checkData(1,3,'ready') - # tdSql.checkData(2,1,'%s:6230'%self.host) - # tdSql.checkData(2,2,'follower') - # tdSql.checkData(2,3,'ready') - - # def check3mnode3off(self): - # count=0 - # while count < 10: - # time.sleep(1) - # tdSql.query("show mnodes;") - # if tdSql.checkRows(3) : - # print("mnode is three nodes") - # if tdSql.queryResult[0][2]=='leader' : - # if tdSql.queryResult[2][2]=='offline': - # if tdSql.queryResult[1][2]=='follower': - # print("stop mnodes on dnode 3 successfully in 10s") - # break - # count+=1 - # else: - # print("stop mnodes on dnode 3 failed in 10s") - # return -1 - # tdSql.error("drop mnode on dnode 3;") - # tdSql.query("show mnodes;") - # tdSql.checkRows(3) - # tdSql.checkData(0,1,'%s:6030'%self.host) - # tdSql.checkData(0,2,'leader') - # tdSql.checkData(0,3,'ready') - # tdSql.checkData(1,1,'%s:6130'%self.host) - # tdSql.checkData(1,2,'follower') - # tdSql.checkData(1,3,'ready') - # tdSql.checkData(2,1,'%s:6230'%self.host) - # tdSql.checkData(2,2,'offline') - # tdSql.checkData(2,3,'ready') - +cluster = ConfigureyCluster() \ No newline at end of file diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index e1f66ceacb..69b9c3d879 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -159,13 +159,10 @@ class TDTestCase: tdSql.query("show mnodes;") tdSql.checkRows(3) tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'leader') tdSql.checkData(0,3,'ready') tdSql.checkData(1,1,'%s:6130'%self.host) - tdSql.checkData(1,2,'offline') tdSql.checkData(1,3,'ready') tdSql.checkData(2,1,'%s:6230'%self.host) - tdSql.checkData(2,2,'follower') tdSql.checkData(2,3,'ready') def check3mnode3off(self): @@ -265,13 +262,6 @@ class TDTestCase: while stopcount <= 2: for i in range(dnodenumber): tdDnodes[i].stoptaosd() - # if i == 1 : - # self.check3mnode2off() - # elif i == 2 : - # self.check3mnode3off() - # elif i == 0: - # self.check3mnode1off() - tdDnodes[i].starttaosd() # self.check3mnode() stopcount+=1 diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 19a67e924c..7d9c8ba633 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -112,7 +112,7 @@ python3 ./test.py -f 2-query/twa.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py -python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 # python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 4ca16e7a57..4414265811 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -59,11 +59,12 @@ if __name__ == "__main__": logSql = True stop = 0 restart = False - dnodeNums = 1 + dnodeNums = 5 + mnodeNums = 0 updateCfgDict = {} execCmd = "" - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums']) + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -79,7 +80,8 @@ if __name__ == "__main__": tdLog.printNoPrefix('-d update cfg dict, base64 json str') tdLog.printNoPrefix('-k not kill valgrind processer') tdLog.printNoPrefix('-e eval str to run') - tdLog.printNoPrefix('-N create dnodes numbers clusters') + tdLog.printNoPrefix('-N create dnodes numbers in clusters') + tdLog.printNoPrefix('-M create mnode numbers in clusters') sys.exit(0) @@ -133,6 +135,9 @@ if __name__ == "__main__": if key in ['-N', '--dnodeNums']: dnodeNums = value + if key in ['-M', '--mnodeNums']: + mnodeNums = value + if not execCmd == "": tdDnodes.init(deployPath) print(execCmd) @@ -244,9 +249,8 @@ if __name__ == "__main__": tdDnodes.start(1) tdCases.logSql(logSql) else : - print("start cluster and dnodes number") - - dnodeslist = cluster.configure_cluster(dnodes_nums=dnodeNums,independent=True) + tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) From c93d93e2d5ebffe2d56f4b57640c9f0d8e9c7fda Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 18:23:51 +0800 Subject: [PATCH 48/55] enh: add stbs/tbs/dbs to monitor info --- include/libs/monitor/monitor.h | 3 +++ source/dnode/mnode/impl/src/mndMain.c | 3 +++ source/libs/monitor/src/monMain.c | 3 +++ source/libs/monitor/src/monMsg.c | 6 ++++++ 4 files changed, 15 insertions(+) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 39e8042b93..9e7aa4e27f 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -109,6 +109,9 @@ typedef struct { char version[MON_VER_LEN]; float master_uptime; // day int32_t monitor_interval; // sec + int32_t dbs_total; + int32_t stbs_total; + int64_t tbs_total; int32_t vgroups_total; int32_t vgroups_alive; int32_t vnodes_total; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index dede1c45e6..7b16adfb73 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -632,6 +632,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version)); pClusterInfo->monitor_interval = tsMonitorInterval; pClusterInfo->connections_total = mndGetNumOfConnections(pMnode); + pClusterInfo->dbs_total = sdbGetSize(pSdb, SDB_DB); + pClusterInfo->stbs_total = sdbGetSize(pSdb, SDB_STB); void *pIter = NULL; while (1) { @@ -681,6 +683,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr if (pIter == NULL) break; pClusterInfo->vgroups_total++; + pClusterInfo->tbs_total += pVgroup->numOfTables; SMonVgroupDesc desc = {0}; desc.vgroup_id = pVgroup->vgId; diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index bf857ad718..4d094e5931 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -201,6 +201,9 @@ static void monGenClusterJson(SMonInfo *pMonitor) { tjsonAddStringToObject(pJson, "version", pInfo->version); tjsonAddDoubleToObject(pJson, "master_uptime", pInfo->master_uptime); tjsonAddDoubleToObject(pJson, "monitor_interval", pInfo->monitor_interval); + tjsonAddDoubleToObject(pJson, "dbs_total", pInfo->dbs_total); + tjsonAddDoubleToObject(pJson, "tbs_total", pInfo->tbs_total); + tjsonAddDoubleToObject(pJson, "stbs_total", pInfo->stbs_total); tjsonAddDoubleToObject(pJson, "vgroups_total", pInfo->vgroups_total); tjsonAddDoubleToObject(pJson, "vgroups_alive", pInfo->vgroups_alive); tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total); diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c index a041b582a9..ced493d5b7 100644 --- a/source/libs/monitor/src/monMsg.c +++ b/source/libs/monitor/src/monMsg.c @@ -102,6 +102,9 @@ int32_t tEncodeSMonClusterInfo(SEncoder *encoder, const SMonClusterInfo *pInfo) if (tEncodeCStr(encoder, pInfo->version) < 0) return -1; if (tEncodeFloat(encoder, pInfo->master_uptime) < 0) return -1; if (tEncodeI32(encoder, pInfo->monitor_interval) < 0) return -1; + if (tEncodeI32(encoder, pInfo->dbs_total) < 0) return -1; + if (tEncodeI32(encoder, pInfo->stbs_total) < 0) return -1; + if (tEncodeI64(encoder, pInfo->tbs_total) < 0) return -1; if (tEncodeI32(encoder, pInfo->vgroups_total) < 0) return -1; if (tEncodeI32(encoder, pInfo->vgroups_alive) < 0) return -1; if (tEncodeI32(encoder, pInfo->vnodes_total) < 0) return -1; @@ -130,6 +133,9 @@ int32_t tDecodeSMonClusterInfo(SDecoder *decoder, SMonClusterInfo *pInfo) { if (tDecodeCStrTo(decoder, pInfo->version) < 0) return -1; if (tDecodeFloat(decoder, &pInfo->master_uptime) < 0) return -1; if (tDecodeI32(decoder, &pInfo->monitor_interval) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->dbs_total) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->stbs_total) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->tbs_total) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vgroups_total) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vgroups_alive) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vnodes_total) < 0) return -1; From 9462c80508612104016b463d3b1bda689228db9e Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 23 Jun 2022 18:25:49 +0800 Subject: [PATCH 49/55] test: modify testcase of muti-mnodes --- tests/pytest/util/cluster.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py index 6d3be8c7a9..6eb78f0771 100644 --- a/tests/pytest/util/cluster.py +++ b/tests/pytest/util/cluster.py @@ -50,12 +50,10 @@ class ConfigureyCluster: dnode.addExtraCfg("fqdn", f"{hostname}") dnode.addExtraCfg("serverPort", f"{self.startPort + (num-1)*self.portStep}") dnode.addExtraCfg("secondEp", f"{hostname}:{startPort_sec}") - print(num,self.mnodeNums) # configure dnoe of independent mnodes if num <= self.mnodeNums and self.mnodeNums != 0 : dnode.addExtraCfg("supportVnodes", 0) - print("-----------------88888123131231320999999") # print(dnode) self.dnodes.append(dnode) return self.dnodes From 800098a79536aff581b5d16f67ef2ba964b9ab52 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 18:54:07 +0800 Subject: [PATCH 50/55] fix(query): fix syntax error on windows platform. --- source/libs/scalar/test/scalar/scalarTests.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index e49c11c390..07440e7435 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -86,7 +86,8 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s if (newBlock) { SSDataBlock *res = createDataBlock(); - SColumnInfoData idata = {.info = *colInfo}; + SColumnInfoData idata = {0}; + idata.info = *colInfo; colInfoDataEnsureCapacity(&idata, rows); blockDataAppendColInfo(res, &idata); @@ -263,9 +264,6 @@ void scltMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNod *pNode = (SNode *)onode; } - - - } TEST(constantTest, bigint_add_bigint) { From df1f4cc53faa22869e1ba473652f399d8972a2ee Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 19:34:10 +0800 Subject: [PATCH 51/55] fix compilation error --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 690319a4cf..5f96b9e287 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -561,7 +561,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { // check for overflow if (IS_FLOAT_TYPE(type) && (isinf(pSumRes->dsum) || isnan(pSumRes->dsum))) { - numOfElems = 0; + numOfElem = 0; } _sum_over: From d9b88d6c4808cd8e53db17768c9c081693f9fdb0 Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 23 Jun 2022 20:46:09 +0800 Subject: [PATCH 52/55] test:modify test.py and set dnode number default value --- tests/system-test/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 4414265811..7d78f5ab21 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -59,7 +59,7 @@ if __name__ == "__main__": logSql = True stop = 0 restart = False - dnodeNums = 5 + dnodeNums = 1 mnodeNums = 0 updateCfgDict = {} execCmd = "" From 9f0d89d6ff882ab654c51ba786266abc1a69f756 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 21:09:50 +0800 Subject: [PATCH 53/55] enh: add stbinfo to minotor --- include/dnode/mnode/mnode.h | 4 ++- include/libs/monitor/monitor.h | 10 +++++++ source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mnode/impl/src/mndMain.c | 27 +++++++++++++++++-- source/libs/monitor/src/monMain.c | 22 +++++++++++++++ source/libs/monitor/src/monMsg.c | 30 +++++++++++++++++++++ tests/script/tmp/monitor.sim | 2 ++ 7 files changed, 93 insertions(+), 4 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 2509c10601..70056783ea 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -73,10 +73,12 @@ void mndStop(SMnode *pMnode); * @param pMnode The mnode object. * @param pCluster * @param pVgroup + * @param pStbInfo * @param pGrant * @return int32_t 0 for success, -1 for failure. */ -int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pCluster, SMonVgroupInfo *pVgroup, SMonGrantInfo *pGrant); +int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo); /** * @brief Get mnode loads for status msg. diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 9e7aa4e27f..41d80c1a83 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -138,6 +138,15 @@ typedef struct { SArray *vgroups; // array of SMonVgroupDesc } SMonVgroupInfo; +typedef struct { + char stb_name[TSDB_TABLE_NAME_LEN]; + char database_name[TSDB_DB_NAME_LEN]; +} SMonStbDesc; + +typedef struct { + SArray *stbs; // array of SMonStbDesc +} SMonStbInfo; + typedef struct { int32_t expire_time; int64_t timeseries_used; @@ -147,6 +156,7 @@ typedef struct { typedef struct { SMonClusterInfo cluster; SMonVgroupInfo vgroup; + SMonStbInfo stb; SMonGrantInfo grant; SMonSysInfo sys; SMonLogs log; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index d44a7d79bf..f9c54674f3 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -17,7 +17,7 @@ #include "mmInt.h" void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *pInfo) { - mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->grant); + mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->stb, &pInfo->grant); } int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 7b16adfb73..000e1041d0 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -614,7 +614,7 @@ int64_t mndGenerateUid(char *name, int32_t len) { } int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo) { + SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) { if (mndAcquireRpcRef(pMnode) != 0) return -1; SSdb *pSdb = pMnode->pSdb; @@ -623,7 +623,9 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc)); pClusterInfo->mnodes = taosArrayInit(sdbGetSize(pSdb, SDB_MNODE), sizeof(SMonMnodeDesc)); pVgroupInfo->vgroups = taosArrayInit(sdbGetSize(pSdb, SDB_VGROUP), sizeof(SMonVgroupDesc)); - if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL) { + pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc)); + if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL || + pStbInfo->stbs == NULL) { mndReleaseRpcRef(pMnode); return -1; } @@ -714,6 +716,27 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr sdbRelease(pSdb, pVgroup); } + // stb info + pIter = NULL; + while (1) { + SStbObj *pStb = NULL; + pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb); + if (pIter == NULL) break; + + SMonStbDesc desc = {0}; + + SName name1 = {0}; + tNameFromString(&name1, pStb->db, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + tNameGetDbName(&name1, desc.database_name); + + SName name2 = {0}; + tNameFromString(&name2, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + tstrncpy(desc.stb_name, tNameGetTableName(&name2), TSDB_TABLE_NAME_LEN); + + taosArrayPush(pStbInfo->stbs, &desc); + sdbRelease(pSdb, pStb); + } + // grant info pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f; pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed; diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 4d094e5931..d387a432e1 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -280,6 +280,27 @@ static void monGenVgroupJson(SMonInfo *pMonitor) { } } +static void monGenStbJson(SMonInfo *pMonitor) { + SMonStbInfo *pInfo = &pMonitor->mmInfo.stb; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + SJson *pJson = tjsonAddArrayToObject(pMonitor->pJson, "stb_infos"); + if (pJson == NULL) return; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->stbs); ++i) { + SJson *pStbJson = tjsonCreateObject(); + if (pStbJson == NULL) continue; + if (tjsonAddItemToArray(pJson, pStbJson) != 0) { + tjsonDelete(pStbJson); + continue; + } + + SMonStbDesc *pStbDesc = taosArrayGet(pInfo->stbs, i); + tjsonAddStringToObject(pStbJson, "stb_name", pStbDesc->stb_name); + tjsonAddStringToObject(pStbJson, "database_name", pStbDesc->database_name); + } +} + static void monGenGrantJson(SMonInfo *pMonitor) { SMonGrantInfo *pInfo = &pMonitor->mmInfo.grant; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; @@ -527,6 +548,7 @@ void monSendReport() { monGenBasicJson(pMonitor); monGenClusterJson(pMonitor); monGenVgroupJson(pMonitor); + monGenStbJson(pMonitor); monGenGrantJson(pMonitor); monGenDnodeJson(pMonitor); monGenDiskJson(pMonitor); diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c index ced493d5b7..8fa7e88605 100644 --- a/source/libs/monitor/src/monMsg.c +++ b/source/libs/monitor/src/monMsg.c @@ -209,6 +209,32 @@ int32_t tDecodeSMonVgroupInfo(SDecoder *decoder, SMonVgroupInfo *pInfo) { return 0; } +int32_t tEncodeSMonStbInfo(SEncoder *encoder, const SMonStbInfo *pInfo) { + if (tEncodeI32(encoder, taosArrayGetSize(pInfo->stbs)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->stbs); ++i) { + SMonStbDesc *pDesc = taosArrayGet(pInfo->stbs, i); + if (tEncodeCStr(encoder, pDesc->stb_name) < 0) return -1; + if (tEncodeCStr(encoder, pDesc->database_name) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSMonStbInfo(SDecoder *decoder, SMonStbInfo *pInfo) { + int32_t arraySize = 0; + if (tDecodeI32(decoder, &arraySize) < 0) return -1; + + pInfo->stbs = taosArrayInit(arraySize, sizeof(SMonStbDesc)); + if (pInfo->stbs == NULL) return -1; + + for (int32_t i = 0; i < arraySize; ++i) { + SMonStbDesc desc = {0}; + if (tDecodeCStrTo(decoder, desc.stb_name) < 0) return -1; + if (tDecodeCStrTo(decoder, desc.database_name) < 0) return -1; + taosArrayPush(pInfo->stbs, &desc); + } + return 0; +} + int32_t tEncodeSMonGrantInfo(SEncoder *encoder, const SMonGrantInfo *pInfo) { if (tEncodeI32(encoder, pInfo->expire_time) < 0) return -1; if (tEncodeI64(encoder, pInfo->timeseries_used) < 0) return -1; @@ -230,6 +256,7 @@ int32_t tSerializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeSMonClusterInfo(&encoder, &pInfo->cluster) < 0) return -1; if (tEncodeSMonVgroupInfo(&encoder, &pInfo->vgroup) < 0) return -1; + if (tEncodeSMonStbInfo(&encoder, &pInfo->stb) < 0) return -1; if (tEncodeSMonGrantInfo(&encoder, &pInfo->grant) < 0) return -1; if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; @@ -247,6 +274,7 @@ int32_t tDeserializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeSMonClusterInfo(&decoder, &pInfo->cluster) < 0) return -1; if (tDecodeSMonVgroupInfo(&decoder, &pInfo->vgroup) < 0) return -1; + if (tDecodeSMonStbInfo(&decoder, &pInfo->stb) < 0) return -1; if (tDecodeSMonGrantInfo(&decoder, &pInfo->grant) < 0) return -1; if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; @@ -261,9 +289,11 @@ void tFreeSMonMmInfo(SMonMmInfo *pInfo) { taosArrayDestroy(pInfo->cluster.mnodes); taosArrayDestroy(pInfo->cluster.dnodes); taosArrayDestroy(pInfo->vgroup.vgroups); + taosArrayDestroy(pInfo->stb.stbs); pInfo->cluster.mnodes = NULL; pInfo->cluster.dnodes = NULL; pInfo->vgroup.vgroups = NULL; + pInfo->stb.stbs = NULL; pInfo->log.logs = NULL; } diff --git a/tests/script/tmp/monitor.sim b/tests/script/tmp/monitor.sim index ba98bec2d0..036d4cc6db 100644 --- a/tests/script/tmp/monitor.sim +++ b/tests/script/tmp/monitor.sim @@ -12,6 +12,8 @@ sql connect print =============== show dnodes sleep 2000 sql create database db vgroups 2; +sql use db; +sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd"; sleep 2000 print =============== create drop qnode 1 From 3d9ffc42e87ccadaf8dea395b13c21352f4bb030 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 23 Jun 2022 21:16:16 +0800 Subject: [PATCH 54/55] refactor(sync): refactor wal abstraction --- include/libs/sync/sync.h | 6 +- source/libs/sync/inc/syncRaftLog.h | 2 +- source/libs/sync/src/syncAppendEntries.c | 37 ++--- source/libs/sync/src/syncMain.c | 18 ++- source/libs/sync/src/syncRaftLog.c | 159 ++++++++++++++++----- source/libs/sync/src/syncSnapshot.c | 3 +- source/libs/sync/test/syncRaftLogTest2.cpp | 33 ++++- source/libs/sync/test/syncRaftLogTest3.cpp | 4 +- 8 files changed, 187 insertions(+), 75 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index a0c07f0c09..2bf49fa006 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -157,13 +157,13 @@ typedef struct SSyncLogStore { SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); // refactor, log[0 .. n] ==> log[m .. n] - int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); - int32_t (*syncLogResetBeginIndex)(struct SSyncLogStore* pLogStore); + // int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); + SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore); bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore); int32_t (*syncLogEntryCount)(struct SSyncLogStore* pLogStore); - // bool (*syncLogInRange)(struct SSyncLogStore* pLogStore, SyncIndex index); + int32_t (*syncLogRestoreFromSnapshot)(struct SSyncLogStore* pLogStore, SyncIndex index); SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore); diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index bc848d217c..f124867097 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -32,7 +32,7 @@ typedef struct SSyncLogStoreData { SSyncNode* pSyncNode; SWal* pWal; SWalReadHandle* pWalHandle; - SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 + // SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 } SSyncLogStoreData; SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 073fdd104d..47d0976aca 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -420,44 +420,26 @@ static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) { // prevLogIndex == -1 static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) { if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) { - if (gRaftDetailLog) { - sTrace("syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld", pMsg->prevLogIndex); - } return true; } SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); if (pMsg->prevLogIndex > myLastIndex) { - if (gRaftDetailLog) { - sTrace("syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld", pMsg->prevLogIndex, - myLastIndex); - } + sDebug("vgId:%d sync log not ok, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); return false; } SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); if (myPreLogTerm == SYNC_TERM_INVALID) { - sError("vgId:%d sync get pre term error, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d sync log not ok2, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); return false; } if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) { - if (gRaftDetailLog) { - sTrace( - "syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " - "myPreLogTerm:%lu", - pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); - } return true; } - if (gRaftDetailLog) { - sTrace( - "syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " - "myPreLogTerm:%lu", - pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); - } - + sDebug("vgId:%d sync log not ok3, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); return false; } @@ -528,7 +510,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition0 = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && syncNodeHasSnapshot(ths); bool condition1 = - condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex); + condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex); // donot use syncLogEntryCount!!! use isEmpty bool condition2 = condition0 && (ths->pLogStore->syncLogLastIndex(ths->pLogStore) <= snapshot.lastApplyIndex) && (pMsg->prevLogIndex > myLastIndex); bool condition3 = condition0 && (pMsg->prevLogIndex < snapshot.lastApplyIndex); @@ -574,10 +556,13 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && (pMsg->prevLogIndex <= ths->commitIndex); if (condition) { - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d", - pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); - syncNodeEventLog(ths, logBuf); + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d", pMsg->prevLogIndex, + pMsg->prevLogTerm, pMsg->dataLen); + syncNodeEventLog(ths, logBuf); + } while (0); SyncIndex matchIndex = ths->commitIndex; bool hasAppendEntries = pMsg->dataLen > 0; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 9a1b5e687e..8bbee65369 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1962,17 +1962,21 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { taosMemoryFree(pPreEntry); return preTerm; } else { - if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); - if (snapshot.lastApplyIndex == preIndex) { - return snapshot.lastApplyTerm; - } + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; + if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + if (snapshot.lastApplyIndex == preIndex) { + return snapshot.lastApplyTerm; } } } + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "sync node get pre term error, index:%ld", index); + syncNodeErrorLog(pSyncNode, logBuf); + } while (0); + return SYNC_TERM_INVALID; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 89fd613307..cc606f4df8 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -18,7 +18,8 @@ #include "syncRaftStore.h" // refactor, log[0 .. n] ==> log[m .. n] -static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); +static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex); +// static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); @@ -44,6 +45,7 @@ static int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncI static SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); // refactor, log[0 .. n] ==> log[m .. n] +/* static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex) { // if beginIndex == 0, donot need call this funciton ASSERT(beginIndex > 0); @@ -56,19 +58,30 @@ static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex b walRestoreFromSnapshot(pWal, beginIndex - 1); return 0; } +*/ + +static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex) { + ASSERT(snapshotIndex >= 0); + + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + walRestoreFromSnapshot(pWal, snapshotIndex); + return 0; +} static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - return pData->beginIndex; + SyncIndex firstVer = walGetFirstVer(pWal); + return firstVer; } static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); } static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { - SyncIndex beginIndex = raftLogBeginIndex(pLogStore); - SyncIndex endIndex = raftLogEndIndex(pLogStore); - return (endIndex < beginIndex); + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + return walIsEmpty(pWal); } static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { @@ -95,23 +108,8 @@ static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; SyncIndex lastVer = walGetLastVer(pWal); - SyncIndex firstVer = walGetFirstVer(pWal); - if (lastVer < firstVer) { - // no record - lastIndex = -1; - - } else { - if (firstVer >= 0) { - lastIndex = lastVer; - } else if (firstVer == -1) { - lastIndex = -1; - } else { - ASSERT(0); - } - } - - return lastIndex; + return lastVer; } static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { @@ -121,6 +119,26 @@ static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { return lastVer + 1; } +static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + if (walIsEmpty(pWal)) { + return 0; + } else { + SSyncRaftEntry* pLastEntry; + int32_t code = raftLogGetLastEntry(pLogStore, &pLastEntry); + ASSERT(code == 0); + ASSERT(pLastEntry != NULL); + + SyncTerm lastTerm = pLastEntry->term; + taosMemoryFree(pLastEntry); + return lastTerm; + } + + return 0; +} + +/* static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { SyncTerm lastTerm = 0; if (raftLogEntryCount(pLogStore) == 0) { @@ -136,6 +154,7 @@ static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { } return lastTerm; } +*/ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; @@ -302,6 +321,25 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn return code; } +static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + ASSERT(ppLastEntry != NULL); + + *ppLastEntry = NULL; + if (walIsEmpty(pWal)) { + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; + } else { + SyncIndex lastIndex = raftLogLastIndex(pLogStore); + int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); + return code; + } + + return -1; +} + +/* static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) { *ppLastEntry = NULL; if (raftLogEntryCount(pLogStore) == 0) { @@ -311,6 +349,7 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); return code; } +*/ //------------------------------- SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { @@ -328,15 +367,17 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pData->pWalHandle = walOpenReadHandle(pData->pWal); ASSERT(pData->pWalHandle != NULL); - SyncIndex firstVer = walGetFirstVer(pData->pWal); - SyncIndex lastVer = walGetLastVer(pData->pWal); - if (firstVer >= 0) { - pData->beginIndex = firstVer; - } else if (firstVer == -1) { - pData->beginIndex = lastVer + 1; - } else { - ASSERT(0); - } + /* + SyncIndex firstVer = walGetFirstVer(pData->pWal); + SyncIndex lastVer = walGetLastVer(pData->pWal); + if (firstVer >= 0) { + pData->beginIndex = firstVer; + } else if (firstVer == -1) { + pData->beginIndex = lastVer + 1; + } else { + ASSERT(0); + } + */ pLogStore->appendEntry = logStoreAppendEntry; pLogStore->getEntry = logStoreGetEntry; @@ -346,7 +387,8 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->updateCommitIndex = logStoreUpdateCommitIndex; pLogStore->getCommitIndex = logStoreGetCommitIndex; - pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex; + // pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex; + pLogStore->syncLogRestoreFromSnapshot = raftLogRestoreFromSnapshot; pLogStore->syncLogBeginIndex = raftLogBeginIndex; pLogStore->syncLogEndIndex = raftLogEndIndex; pLogStore->syncLogIsEmpty = raftLogIsEmpty; @@ -538,6 +580,7 @@ SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore) { return pEntry; } +/* cJSON* logStore2Json(SSyncLogStore* pLogStore) { char u64buf[128] = {0}; SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; @@ -584,6 +627,57 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); return pJson; } +*/ + +cJSON* logStore2Json(SSyncLogStore* pLogStore) { + char u64buf[128] = {0}; + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%ld", beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%ld", endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); + + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "pEntries", pEntries); + + if (!raftLogIsEmpty(pLogStore)) { + for (SyncIndex i = beginIndex; i <= endIndex; ++i) { + SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i); + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + syncEntryDestory(pEntry); + } + } + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); + return pJson; +} char* logStore2Str(SSyncLogStore* pLogStore) { cJSON* pJson = logStore2Json(pLogStore); @@ -603,7 +697,8 @@ cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); cJSON_AddStringToObject(pRoot, "pWal", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%ld", beginIndex); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); SyncIndex endIndex = raftLogEndIndex(pLogStore); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index ed9d62fd14..4fc0128400 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -545,7 +545,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex; } - pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); + // pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); + pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, pMsg->lastIndex); // maybe update lastconfig if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) { diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index bd6fa7c2c3..e3ceae0615 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -113,7 +113,8 @@ void test2() { pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); pSyncNode->pLogStore = pLogStore; - pLogStore->syncLogSetBeginIndex(pLogStore, 5); + //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); if (gAssert) { @@ -228,7 +229,8 @@ void test4() { assert(pLogStore); pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore); - pLogStore->syncLogSetBeginIndex(pLogStore, 5); + //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); for (int i = 5; i <= 9; ++i) { int32_t dataLen = 10; @@ -289,7 +291,8 @@ void test5() { assert(pLogStore); pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); - pLogStore->syncLogSetBeginIndex(pLogStore, 5); + //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); for (int i = 5; i <= 9; ++i) { int32_t dataLen = 10; @@ -363,7 +366,8 @@ void test6() { assert(pLogStore); pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest6 ----- ", pLogStore); - pLogStore->syncLogSetBeginIndex(pLogStore, 5); + // pLogStore->syncLogSetBeginIndex(pLogStore, 5); + pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); for (int i = 5; i <= 9; ++i) { int32_t dataLen = 10; @@ -405,14 +409,32 @@ void test6() { assert(pLogStore->syncLogLastTerm(pLogStore) == 0); } + do { + SyncIndex firstVer = walGetFirstVer(pWal); + SyncIndex lastVer = walGetLastVer(pWal); + bool isEmpty = walIsEmpty(pWal); + printf("before -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty); + } while (0); + logStoreDestory(pLogStore); cleanup(); + + // restart init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); pSyncNode->pLogStore = pLogStore; + + + do { + SyncIndex firstVer = walGetFirstVer(pWal); + SyncIndex lastVer = walGetLastVer(pWal); + bool isEmpty = walIsEmpty(pWal); + printf("after -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty); + } while (0); + logStoreLog2((char*)"\n\n\ntest6 restart ----- ", pLogStore); if (gAssert) { @@ -432,17 +454,20 @@ void test6() { int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; + gRaftDetailLog = true; if (argc == 2) { gAssert = atoi(argv[1]); } sTrace("gAssert : %d", gAssert); +/* test1(); test2(); test3(); test4(); test5(); +*/ test6(); return 0; diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index c81a6d3ca9..302e29a091 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -312,7 +312,8 @@ void test5() { pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); - pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6); + //pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6); + pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, 5); for (int i = 6; i <= 10; ++i) { int32_t dataLen = 10; SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); @@ -372,6 +373,7 @@ void test5() { int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; + gRaftDetailLog = true; if (argc == 2) { gAssert = atoi(argv[1]); From f2f5d68f3b25fd1dc0a874fdc7254a22c8da7b50 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 23 Jun 2022 22:05:00 +0800 Subject: [PATCH 55/55] fix(stream): memory leak --- include/common/tdatablock.h | 8 +-- include/libs/stream/tstream.h | 2 + source/common/src/tdatablock.c | 72 ++++++++++++++++++------- source/dnode/vnode/src/tq/tqExec.c | 1 + source/dnode/vnode/src/tq/tqOffset.c | 4 +- source/dnode/vnode/src/tq/tqRead.c | 10 ++-- source/libs/executor/inc/executorimpl.h | 1 - source/libs/executor/src/executorimpl.c | 34 ++++++------ source/libs/executor/src/scanoperator.c | 20 +++---- source/libs/stream/inc/streamInc.h | 2 +- source/libs/stream/src/streamDispatch.c | 6 ++- source/libs/stream/src/streamExec.c | 45 +++++++++------- 12 files changed, 128 insertions(+), 77 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 6204b9dcfc..2a0d4e7ff6 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -71,8 +71,8 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); #define colDataGetData(p1_, r_) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_)) -#define IS_JSON_NULL(type, data) ((type) == TSDB_DATA_TYPE_JSON && \ - (*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data))) +#define IS_JSON_NULL(type, data) \ + ((type) == TSDB_DATA_TYPE_JSON && (*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data))) static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { if (!pColumnInfoData->hasNull) { @@ -186,7 +186,8 @@ int32_t getJsonValueLen(const char* data); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); -int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo); +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, + const SDataBlockInfo* pBlockInfo); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); @@ -222,6 +223,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); +int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); SSDataBlock* createDataBlock(); int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 7673e73035..a47810e7c1 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -344,10 +344,12 @@ static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBloc if (pTask->sinkType == TASK_SINK__TABLE) { ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks); + taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock); taosFreeQitem(pBlock); } else if (pTask->sinkType == TASK_SINK__SMA) { ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); + taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock); taosFreeQitem(pBlock); } else { ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 8317c6c79d..593f8c5c0b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -263,7 +263,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui pColumnInfoData->varmeta.length = len + oldLen; } else { if (finalNumOfRows > *capacity) { - ASSERT(finalNumOfRows*pColumnInfoData->info.bytes); + ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; @@ -293,7 +293,8 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui return numOfRow1 + numOfRow2; } -int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo) { +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, + const SDataBlockInfo* pBlockInfo) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRows <= 0) { return numOfRows; @@ -327,9 +328,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p return 0; } -size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { - return taosArrayGetSize(pBlock->pDataBlock); -} +size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSize(pBlock->pDataBlock); } size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } @@ -396,7 +395,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd int32_t pageSize) { ASSERT(pBlock != NULL && stopIndex != NULL); - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; int32_t bitmapChar = 1; @@ -512,7 +511,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { // write the number of rows *(uint32_t*)buf = pBlock->info.rows; - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; char* pStart = buf + sizeof(uint32_t); @@ -542,7 +541,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { pBlock->info.rows = *(int32_t*)buf; - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); const char* pStart = buf + sizeof(uint32_t); for (int32_t i = 0; i < numOfCols; ++i) { @@ -734,7 +733,7 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, int32_t tupleIndex) { int32_t code = 0; - size_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = &pDstCols[i]; @@ -794,7 +793,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { int32_t rows = pDataBlock->info.rows; - size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData)); if (pCols == NULL) { @@ -902,7 +901,6 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { } else { // var data type } } else if (numOfCols == 2) { - } } @@ -1103,14 +1101,14 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { } } -static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo *pBlockInfo, uint32_t numOfRows) { +static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows) { ASSERT(numOfRows > 0 && pBlockInfo->capacity >= pBlockInfo->rows); if (numOfRows < pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } // todo temp disable it -// ASSERT(pColumn->info.bytes != 0); + // ASSERT(pColumn->info.bytes != 0); int32_t existedRows = pBlockInfo->rows; @@ -1141,7 +1139,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo * if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - + memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); pColumn->pData = tmp; } @@ -1197,6 +1195,40 @@ void* blockDataDestroy(SSDataBlock* pBlock) { taosMemoryFreeClear(pBlock); return NULL; } +int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { + ASSERT(src != NULL); + + dst->info = src->info; + dst->info.rows = 0; + dst->info.capacity = 0; + + size_t numOfCols = taosArrayGetSize(src->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(src->pDataBlock, i); + SColumnInfoData colInfo = {.hasNull = true, .info = p->info}; + blockDataAppendColInfo(dst, &colInfo); + } + + int32_t code = blockDataEnsureCapacity(dst, src->info.rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return -1; + } + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i); + if (pSrc->pData == NULL) { + continue; + } + + colDataAssign(pDst, pSrc, src->info.rows, &src->info); + } + + dst->info.rows = src->info.rows; + dst->info.capacity = src->info.rows; + return 0; +} SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { if (pDataBlock == NULL) { @@ -1272,7 +1304,7 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoDat } // todo disable it temporarily -// ASSERT(pColInfoData->info.type != 0); + // ASSERT(pColInfoData->info.type != 0); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { pBlock->info.hasVarCol = true; } @@ -1284,7 +1316,7 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoDat SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) { SColumnInfoData col = {.hasNull = true}; col.info.colId = colId; - col.info.type = type; + col.info.type = type; col.info.bytes = bytes; return col; @@ -1552,9 +1584,9 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); - size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); - int32_t rows = pDataBlock->info.rows; + int32_t rows = pDataBlock->info.rows; printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); for (int32_t j = 0; j < rows; j++) { printf("%s |", flag); @@ -1633,8 +1665,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; -// int32_t rowSize = pDataBlock->info.rowSize; -// int64_t groupId = pDataBlock->info.groupId; + // int32_t rowSize = pDataBlock->info.rowSize; + // int64_t groupId = pDataBlock->info.groupId; if (colNum <= 1) { // invalid if only with TS col diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index da052ee3eb..be8fef1249 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -66,6 +66,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp if (qStreamScanSnapshot(task) < 0) { ASSERT(0); } + // set version while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index e5475d7c30..e536e87032 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -44,6 +44,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { } char* fname = buildFileName(pStore->pTq->path); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + taosMemoryFree(fname); if (pFile != NULL) { STqOffsetHead head = {0}; int64_t code; @@ -77,7 +78,6 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { } taosCloseFile(&pFile); - taosMemoryFree(fname); } return pStore; } @@ -102,6 +102,7 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { // TODO file name should be with a version char* fname = buildFileName(pStore->pTq->path); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + taosMemoryFree(fname); if (pFile == NULL) { ASSERT(0); return -1; @@ -140,6 +141,5 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { } // close and rename file taosCloseFile(&pFile); - taosMemoryFree(fname); return 0; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 5e5a4883a9..5d2fb81cf0 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -151,6 +151,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_ int32_t sversion = htonl(pHandle->pBlock->sversion); if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion || pHandle->cachedSchemaSuid != pHandle->msgIter.suid) { + if (pHandle->pSchema) taosMemoryFree(pHandle->pSchema); pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); if (pHandle->pSchema == NULL) { tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table", @@ -161,6 +162,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_ } // this interface use suid instead of uid + if (pHandle->pSchemaWrapper) tDeleteSSchemaWrapper(pHandle->pSchemaWrapper); pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion, true); if (pHandle->pSchemaWrapper == NULL) { tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table", @@ -184,7 +186,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_ while (colMeta < pSchemaWrapper->nCols) { SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); if (code != TSDB_CODE_SUCCESS) { goto FAIL; } @@ -207,7 +209,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_ colNeed++; } else { SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); if (code != TSDB_CODE_SUCCESS) { goto FAIL; } @@ -251,8 +253,8 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_ } return 0; -FAIL: // todo refactor here -// if (*ppCols) taosArrayDestroy(*ppCols); +FAIL: // todo refactor here + // if (*ppCols) taosArrayDestroy(*ppCols); return -1; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 91931c2fd8..57a2c57b16 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -336,7 +336,6 @@ typedef struct SStreamBlockScanInfo { int32_t numOfPseudoExpr; int32_t primaryTsIndex; // primary time stamp slot id - void* pDataReader; SReadHandle readHandle; uint64_t tableUid; // queried super table uid EStreamScanMode scanMode; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 962b7b9c51..afe0a1f8fa 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1341,7 +1341,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR } if (rowRes != NULL) { - int32_t totalRows = pBlock->info.rows; + int32_t totalRows = pBlock->info.rows; SSDataBlock* px = createOneDataBlock(pBlock, true); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); @@ -3954,7 +3954,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } } } - int32_t len = (int32_t)(pStart - (char*)keyBuf); + int32_t len = (int32_t)(pStart - (char*)keyBuf); uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len); @@ -3980,8 +3980,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - tsdbReaderT pDataReader = - doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); + tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); if (pDataReader == NULL && terrno != 0) { pTaskInfo->code = terrno; return NULL; @@ -4007,16 +4006,20 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; return pOperator; + } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); - SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); + SOperatorInfo* pOperator = + createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; return pOperator; + } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; @@ -4031,20 +4034,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pHandle->initTsdbReader) { // for stream ASSERT(pHandle->vnode); - pDataReader = - doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); + pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); } else { // for tq ASSERT(pHandle->meta); getTableList(pHandle->meta, pScanPhyNode, pTableListInfo); } } + +#if 0 if (pDataReader == NULL && terrno != 0) { qDebug("%s pDataReader is NULL", GET_TASKID(pTaskInfo)); // return NULL; } else { qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); } +#endif SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json @@ -4057,9 +4062,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup); return pOperator; + } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; @@ -4070,6 +4077,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } return createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; pTableListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo)); @@ -4343,7 +4351,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* goto _error; } - tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId); + tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId); cleanupQueryTableDataCond(&cond); return pReader; @@ -4487,18 +4495,14 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead (*pTaskInfo)->sql = sql; (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond; (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond; - (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, - &(*pTaskInfo)->tableqinfoList); + (*pTaskInfo)->pRoot = + createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList); + if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; goto _complete; } - if ((*pTaskInfo)->pRoot == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _complete; - } - return code; _complete: diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b521983d8e..07212bc018 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -967,9 +967,9 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { while (tqNextDataBlock(pInfo->streamBlockReader)) { SSDataBlock block = {0}; - uint64_t groupId = 0; - uint64_t uid = 0; - int32_t numOfRows = 0; + uint64_t groupId = 0; + uint64_t uid = 0; + int32_t numOfRows = 0; // todo refactor int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows); @@ -1022,6 +1022,9 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } } + // TODO refactor @liao + taosArrayDestroy(block.pDataBlock); + if (pInfo->pRes->pDataBlock == NULL) { // TODO add log pOperator->status = OP_EXEC_DONE; @@ -1061,12 +1064,11 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; + } else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) { SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp); - if (pResult) { - return pResult->info.rows > 0 ? pResult : NULL; - } - return NULL; + return pResult && pResult->info.rows > 0 ? pResult : NULL; + } else { ASSERT(0); return NULL; @@ -1161,7 +1163,6 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan pInfo->pRes = createResDataBlock(pDescNode); pInfo->pUpdateRes = createResDataBlock(pDescNode); pInfo->pCondition = pScanPhyNode->node.pConditions; - pInfo->pDataReader = pDataReader; pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; pInfo->groupId = 0; @@ -1343,7 +1344,8 @@ static SSDataBlock* buildSysTableMetaBlock() { SSDataBlock* pBlock = createDataBlock(); for (int32_t i = 0; i < pMeta[index].colNum; ++i) { - SColumnInfoData colInfoData = createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); + SColumnInfoData colInfoData = + createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); blockDataAppendColInfo(pBlock, &colInfoData); } diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 9654e21d24..2f41c08354 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -34,7 +34,7 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData); int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); -int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet); +int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet); int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index b5a9e221a7..11cd089606 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -172,7 +172,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->streamBlockType = pBlock->info.type; pRetrieve->numOfRows = htonl(pBlock->info.rows); - int32_t numOfCols = (int32_t) taosArrayGetSize(pBlock->pDataBlock); + int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = 0; @@ -185,7 +185,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis return 0; } -int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { +int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { void* buf = NULL; int32_t code = -1; int32_t blockNum = taosArrayGetSize(data->blocks); @@ -307,6 +307,8 @@ int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) { atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); return -1; } + taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock); + taosFreeQitem(pBlock); tmsgSendReq(pEpSet, &dispatchMsg); return 0; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 6fada59c84..cb7dd241c1 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -53,9 +53,13 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) } // TODO: do we need free memory? - SSDataBlock* outputCopy = createOneDataBlock(output, true); - outputCopy->info.childId = pTask->selfChildId; - taosArrayPush(pRes, outputCopy); + SSDataBlock block = {0}; + assignOneDataBlock(&block, output); + block.info.childId = pTask->selfChildId; + taosArrayPush(pRes, &block); + /*SSDataBlock* outputCopy = createOneDataBlock(output, true);*/ + /*outputCopy->info.childId = pTask->selfChildId;*/ + /*taosArrayPush(pRes, outputCopy);*/ } return 0; } @@ -68,6 +72,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { streamTaskExecImpl(pTask, data, pRes); if (pTask->taskStatus == TASK_STATUS__DROPPING) { + taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock); return NULL; } @@ -82,25 +87,25 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { qRes->blocks = pRes; if (streamTaskOutput(pTask, qRes) < 0) { streamQueueProcessFail(pTask->inputQueue); - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock); taosFreeQitem(qRes); return NULL; } - - int8_t type = ((SStreamQueueItem*)data)->type; - if (type == STREAM_INPUT__TRIGGER) { - blockDataDestroy(((SStreamTrigger*)data)->pBlock); - taosFreeQitem(data); - } else if (type == STREAM_INPUT__DATA_BLOCK) { - taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock); - taosFreeQitem(data); - } else if (type == STREAM_INPUT__DATA_SUBMIT) { - ASSERT(pTask->isDataScan); - streamDataSubmitRefDec((SStreamDataSubmit*)data); - taosFreeQitem(data); - } streamQueueProcessSuccess(pTask->inputQueue); - return taosArrayInit(0, sizeof(SSDataBlock)); + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + } + + int8_t type = ((SStreamQueueItem*)data)->type; + if (type == STREAM_INPUT__TRIGGER) { + blockDataDestroy(((SStreamTrigger*)data)->pBlock); + taosFreeQitem(data); + } else if (type == STREAM_INPUT__DATA_BLOCK) { + taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock); + taosFreeQitem(data); + } else if (type == STREAM_INPUT__DATA_SUBMIT) { + ASSERT(pTask->isDataScan); + streamDataSubmitRefDec((SStreamDataSubmit*)data); + taosFreeQitem(data); } } return pRes; @@ -125,14 +130,14 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) { pRes = streamExecForQall(pTask, pRes); if (pRes == NULL) goto FAIL; - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock); atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE); return 0; } else if (execStatus == TASK_EXEC_STATUS__CLOSING) { continue; } else if (execStatus == TASK_EXEC_STATUS__EXECUTING) { ASSERT(taosArrayGetSize(pRes) == 0); - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock); return 0; } else { ASSERT(0);