From 68dfcfff8ac6852e76d91d7c5fae72ece5f14543 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 20 Jun 2022 20:32:44 +0800 Subject: [PATCH 01/54] refactor: rsma code optimization --- source/dnode/vnode/src/inc/vnodeInt.h | 1 - source/dnode/vnode/src/sma/smaRollup.c | 85 ++++++++++++++------------ 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 70b6e24b07..ed874fecf0 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -245,7 +245,6 @@ struct SVnode { struct STbUidStore { tb_uid_t suid; - tb_uid_t uid; // TODO: just for debugging, remove when uid provided in SSDataBlock SArray* tbUids; SHashObj* uidHash; }; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 1f18f6cb87..51edf66179 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -16,35 +16,12 @@ #include "sma.h" #include "tstream.h" -static FORCE_INLINE int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); -static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); -static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem, - tb_uid_t suid, int8_t level); - -#define SET_RSMA_INFO_ITEM_PARAMS(__idx, __level) \ - if (param->qmsg[__idx]) { \ - pRSmaInfo->items[__idx].pRsmaInfo = pRSmaInfo; \ - pRSmaInfo->items[__idx].taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], &handle); \ - if (!pRSmaInfo->items[__idx].taskInfo) { \ - goto _err; \ - } \ - pRSmaInfo->items[__idx].triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; \ - if (param->maxdelay[__idx] < 1) { \ - int64_t msInterval = \ - convertTimeFromPrecisionToUnit(pRetention[__level].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); \ - pRSmaInfo->items[__idx].maxDelay = msInterval; \ - } else { \ - pRSmaInfo->items[__idx].maxDelay = param->maxdelay[__idx]; \ - } \ - if (pRSmaInfo->items[__idx].maxDelay > TSDB_MAX_ROLLUP_MAX_DELAY) { \ - pRSmaInfo->items[__idx].maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; \ - } \ - pRSmaInfo->items[__idx].level = TSDB_RETENTION_L##__level; \ - pRSmaInfo->items[__idx].tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); \ - if (!pRSmaInfo->items[__idx].tmrHandle) { \ - goto _err; \ - } \ - } +static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); +static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); +static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle, + int8_t idx); +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem, + tb_uid_t suid, int8_t level); struct SRSmaInfoItem { SRSmaInfo *pRsmaInfo; @@ -81,7 +58,7 @@ static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) { void *tdFreeRSmaInfo(SRSmaInfo *pInfo) { if (pInfo) { - for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (pItem->taskInfo) { tdFreeTaskHandle(pItem->taskInfo); @@ -213,6 +190,39 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui return TSDB_CODE_SUCCESS; } +static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *pReadHandle, + int8_t idx) { + SRetention *pRetention = SMA_RETENTION(pSma); + STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); + + if (param->qmsg[idx]) { + pRSmaInfo->items[idx].pRsmaInfo = pRSmaInfo; + pRSmaInfo->items[idx].taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], pReadHandle); + if (!pRSmaInfo->items[idx].taskInfo) { + goto _err; + } + pRSmaInfo->items[idx].triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; + if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { + int64_t msInterval = + convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); + pRSmaInfo->items[idx].maxDelay = msInterval; + } else { + pRSmaInfo->items[idx].maxDelay = param->maxdelay[idx]; + } + if (pRSmaInfo->items[idx].maxDelay > TSDB_MAX_ROLLUP_MAX_DELAY) { + pRSmaInfo->items[idx].maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; + } + pRSmaInfo->items[idx].level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); + pRSmaInfo->items[idx].tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); + if (!pRSmaInfo->items[idx].tmrHandle) { + goto _err; + } + } + return TSDB_CODE_SUCCESS; +_err: + return TSDB_CODE_FAILED; +} + /** * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. * @@ -282,14 +292,14 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { pRSmaInfo->pSma = pSma; pRSmaInfo->suid = pReq->suid; - SRetention *pRetention = SMA_RETENTION(pSma); - STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); + if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) { + goto _err; + } + if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 1) < 0) { + goto _err; + } - SET_RSMA_INFO_ITEM_PARAMS(0, 1); - SET_RSMA_INFO_ITEM_PARAMS(1, 2); - - if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != - TSDB_CODE_SUCCESS) { + if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { goto _err; } else { smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid); @@ -418,7 +428,6 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { if (!pBlock) break; tdUidStorePut(pStore, msgIter.suid, NULL); - pStore->uid = msgIter.uid; // TODO: remove, just for debugging } if (terrno != TSDB_CODE_SUCCESS) return -1; From a1774d913b0f6838382b24343f1bcd6b13128843 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 20 Jun 2022 20:42:28 +0800 Subject: [PATCH 02/54] refactor: code optimization --- source/dnode/vnode/src/sma/smaRollup.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 51edf66179..f70c98bae6 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -196,25 +196,26 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); if (param->qmsg[idx]) { - pRSmaInfo->items[idx].pRsmaInfo = pRSmaInfo; - pRSmaInfo->items[idx].taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], pReadHandle); - if (!pRSmaInfo->items[idx].taskInfo) { + SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); + pItem->pRsmaInfo = pRSmaInfo; + pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], pReadHandle); + if (!pItem->taskInfo) { goto _err; } - pRSmaInfo->items[idx].triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; + pItem->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); - pRSmaInfo->items[idx].maxDelay = msInterval; + pItem->maxDelay = (int32_t)msInterval; } else { - pRSmaInfo->items[idx].maxDelay = param->maxdelay[idx]; + pItem->maxDelay = (int32_t)param->maxdelay[idx]; } - if (pRSmaInfo->items[idx].maxDelay > TSDB_MAX_ROLLUP_MAX_DELAY) { - pRSmaInfo->items[idx].maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; + if (pItem->maxDelay > TSDB_MAX_ROLLUP_MAX_DELAY) { + pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; } - pRSmaInfo->items[idx].level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); - pRSmaInfo->items[idx].tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); - if (!pRSmaInfo->items[idx].tmrHandle) { + pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); + pItem->tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); + if (!pItem->tmrHandle) { goto _err; } } From 14ddf539e04c3e095516e30ce56052a02a0b7140 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 17:21:04 +0800 Subject: [PATCH 03/54] refactor(query): do some internal refactor. --- include/common/tcommon.h | 7 +- include/common/tdatablock.h | 11 +- include/common/tmsg.h | 5 - include/libs/function/function.h | 17 - source/common/src/tdatablock.c | 303 +++++++++++------- source/common/test/commonTests.cpp | 46 +-- source/dnode/mnode/impl/src/mndShow.c | 3 +- source/dnode/vnode/inc/vnode.h | 4 +- source/dnode/vnode/src/tq/tqExec.c | 9 +- source/dnode/vnode/src/tq/tqRead.c | 48 +-- source/dnode/vnode/src/tq/tqSink.c | 4 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 69 ++-- source/dnode/vnode/test/tsdbSmaTest.cpp | 2 +- source/libs/command/src/command.c | 30 +- source/libs/command/src/explain.c | 17 +- source/libs/executor/src/dataDeleter.c | 2 +- source/libs/executor/src/executil.c | 18 +- source/libs/executor/src/executor.c | 2 + source/libs/executor/src/executorimpl.c | 54 ++-- source/libs/executor/src/groupoperator.c | 4 +- source/libs/executor/src/scanoperator.c | 53 ++- source/libs/executor/src/sortoperator.c | 7 +- source/libs/executor/src/tfill.c | 2 +- source/libs/executor/src/timewindowoperator.c | 13 +- source/libs/executor/src/tsort.c | 8 +- source/libs/executor/test/executorTests.cpp | 42 +-- source/libs/executor/test/sortTests.cpp | 24 +- source/libs/function/inc/taggfunction.h | 13 - source/libs/function/src/builtinsimpl.c | 16 +- source/libs/function/src/tudf.c | 43 +-- source/libs/function/test/runUdf.c | 50 ++- .../libs/index/test/index_executor_tests.cpp | 2 +- source/libs/scalar/inc/sclInt.h | 2 +- source/libs/scalar/src/filter.c | 2 +- source/libs/scalar/src/scalar.c | 22 +- source/libs/scalar/src/sclfunc.c | 4 +- source/libs/scalar/src/sclvector.c | 4 +- .../libs/scalar/test/filter/filterTests.cpp | 282 ++++++++-------- .../libs/scalar/test/scalar/scalarTests.cpp | 60 ++-- source/libs/stream/src/streamDispatch.c | 6 +- 40 files changed, 585 insertions(+), 725 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a05287761e..371f182d53 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -66,11 +66,10 @@ typedef struct SDataBlockInfo { int32_t rows; int32_t rowSize; int64_t uid; // the uid of table, from which current data block comes - int64_t blockId; // block id, generated by physical planner + uint16_t blockId; // block id, generated by physical planner uint64_t groupId; // no need to serialize - int16_t numOfCols; int16_t hasVarCol; - int32_t capacity; + uint32_t capacity; // TODO: optimize and remove following int32_t childId; // used for stream, do not serialize EStreamType type; // used for stream, do not serialize @@ -91,7 +90,7 @@ typedef struct SVarColAttr { // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // TODO filter info needs to be removed + SColumnInfo info; // column info bool hasNull; // if current column data has null value. char* pData; // the corresponding block data in memory union { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 709462a744..d65fdc0675 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -185,7 +185,7 @@ int32_t getJsonValueLen(const char* data); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); -int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); @@ -211,7 +211,7 @@ size_t blockDataGetSerialMetaSize(uint32_t numOfCols); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); -int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows); +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); @@ -222,6 +222,11 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); +SSDataBlock* createDataBlock(); +int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); + +SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); +SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index); void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); @@ -235,7 +240,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { - return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock); + return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock); } static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, diff --git a/include/common/tmsg.h b/include/common/tmsg.h index fd9ff3eb65..a84bc3cd03 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -628,11 +628,6 @@ typedef struct { uint8_t scale; } SColumnInfo; -typedef struct { - int64_t uid; - TSKEY key; // last accessed ts, for subscription -} STableIdInfo; - typedef struct STimeWindow { TSKEY skey; TSKEY ekey; diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 0d8ae289cc..c5842559f8 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -172,23 +172,6 @@ typedef struct tExprNode { void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); -typedef struct SAggFunctionInfo { - char name[FUNCTIONS_NAME_MAX_LENGTH]; - int8_t type; // Scalar function or aggregation function - uint32_t functionId; // Function Id - int8_t sFunctionId; // Transfer function for super table query - uint16_t status; - - bool (*init)(SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*addInput)(SqlFunctionCtx *pCtx); - - // finalizer must be called after all exec has been executed to generated final result. - void (*finalize)(SqlFunctionCtx *pCtx); - void (*combine)(SqlFunctionCtx *pCtx); - - int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); -} SAggFunctionInfo; - struct SScalarParam { SColumnInfoData *columnData; SHashObj *pHashFilter; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5594ee9bcf..7856f3a29b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -293,53 +293,61 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in return numOfRow1 + numOfRow2; } -int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows) { +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRows <= 0) { return numOfRows; } + ASSERT(pBlockInfo->capacity >= numOfRows); + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - // Handle the bitmap - char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); - if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } +// if (pColumnInfoData->capacity < numOfRows) { +// char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); +// if (p == NULL) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// +// memset(p, 0, sizeof(int32_t)); +// pColumnInfoData->capacity = numOfRows; +// pColumnInfoData->varmeta.offset = (int32_t*)p; +// } - pColumnInfoData->varmeta.offset = (int32_t*)p; memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); - - if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { - char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pColumnInfoData->pData = tmp; - pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; - } +// if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { +// char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); +// if (tmp == NULL) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// +// pColumnInfoData->pData = tmp; +// pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; +// } memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); pColumnInfoData->varmeta.length = pSource->varmeta.length; } else { - char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } +// if (pColumnInfoData->capacity < numOfRows) { +// char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); +// if (tmp == NULL) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// +// memset(tmp, 0, BitmapLen(numOfRows)); +// pColumnInfoData->nullbitmap = tmp; +// +// int32_t newSize = numOfRows * pColumnInfoData->info.bytes; +// tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); +// if (tmp == NULL) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// +// pColumnInfoData->pData = tmp; +// pColumnInfoData->capacity = numOfRows; +// } - pColumnInfoData->nullbitmap = tmp; memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); - - if (pSource->pData) { - int32_t newSize = numOfRows * pColumnInfoData->info.bytes; - tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pColumnInfoData->pData = tmp; - memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); - } + memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); } pColumnInfoData->hasNull = pSource->hasNull; @@ -348,8 +356,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p } size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { - ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock)); - return pBlock->info.numOfCols; + return taosArrayGetSize(pBlock->pDataBlock); } size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } @@ -359,7 +366,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) return 0; } - if (pDataBlock->info.numOfCols <= 0) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + if (numOfCols <= 0) { return -1; } @@ -383,7 +391,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { assert(pSrc != NULL && pDest != NULL); int32_t capacity = pDest->info.capacity; - for (int32_t i = 0; i < pDest->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDest->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); @@ -399,9 +408,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { size_t blockDataGetSize(const SSDataBlock* pBlock) { assert(pBlock != NULL); - size_t total = 0; - int32_t numOfCols = pBlock->info.numOfCols; - + size_t total = 0; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); total += colDataGetFullLength(pColInfoData, pBlock->info.rows); @@ -416,7 +424,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd int32_t pageSize) { ASSERT(pBlock != NULL && stopIndex != NULL); - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; int32_t bitmapChar = 1; @@ -477,33 +485,24 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 return NULL; } - SSDataBlock* pDst = taosMemoryCalloc(1, sizeof(SSDataBlock)); + SSDataBlock* pDst = createDataBlock(); if (pDst == NULL) { return NULL; } pDst->info = pBlock->info; - pDst->info.rows = 0; - pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i); colInfo.info = pSrcCol->info; - - if (IS_VAR_DATA_TYPE(pSrcCol->info.type)) { - SVarColAttr* pAttr = &colInfo.varmeta; - pAttr->offset = taosMemoryCalloc(rowCount, sizeof(int32_t)); - } else { - colInfo.nullbitmap = taosMemoryCalloc(1, BitmapLen(rowCount)); - colInfo.pData = taosMemoryCalloc(rowCount, colInfo.info.bytes); - } - - taosArrayPush(pDst->pDataBlock, &colInfo); + blockDataAppendColInfo(pDst, &colInfo); } - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + blockDataEnsureCapacity(pDst, rowCount); + + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); @@ -541,7 +540,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { // write the number of rows *(uint32_t*)buf = pBlock->info.rows; - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; char* pStart = buf + sizeof(uint32_t); @@ -571,7 +570,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { pBlock->info.rows = *(int32_t*)buf; - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); const char* pStart = buf + sizeof(uint32_t); for (int32_t i = 0; i < numOfCols; ++i) { @@ -620,7 +619,8 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) pBlock->info.rows = *(int32_t*)buf; pBlock->info.groupId = *(uint64_t*)(buf + sizeof(int32_t)); - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + const char* pStart = buf + sizeof(uint32_t) + sizeof(uint64_t); for (int32_t i = 0; i < numOfCols; ++i) { @@ -666,7 +666,7 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { if (pBlock->info.rowSize == 0) { size_t rowSize = 0; - size_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); rowSize += pColInfo->info.bytes; @@ -693,7 +693,7 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { ASSERT(pBlock != NULL); double rowSize = 0; - size_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); rowSize += pColInfo->info.bytes; @@ -762,7 +762,7 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, int32_t tupleIndex) { int32_t code = 0; - int32_t numOfCols = pSrcBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = &pDstCols[i]; @@ -785,7 +785,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co return TSDB_CODE_SUCCESS; } -static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, int32_t* index) { +static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) { #if 0 for (int32_t i = 0; i < pDataBlock->info.rows; ++i) { int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]); @@ -794,7 +794,8 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB } } #else - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = &pCols[i]; SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); @@ -821,7 +822,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { int32_t rows = pDataBlock->info.rows; - int32_t numOfCols = pDataBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData)); if (pCols == NULL) { @@ -848,7 +849,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { } static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { - int32_t numOfCols = pDataBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); @@ -909,8 +910,10 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { } } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + if (taosArrayGetSize(pOrderInfo) == 1 && (!sortColumnHasNull)) { - if (pDataBlock->info.numOfCols == 1) { + if (numOfCols == 1) { if (!varTypeSort) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0); SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0); @@ -926,7 +929,8 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { return TSDB_CODE_SUCCESS; } else { // var data type } - } else if (pDataBlock->info.numOfCols == 2) { + } else if (numOfCols == 2) { + } } @@ -1120,34 +1124,36 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF void blockDataCleanup(SSDataBlock* pDataBlock) { pDataBlock->info.rows = 0; - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); colInfoDataCleanup(p, pDataBlock->info.capacity); } } -int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows) { - ASSERT(numOfRows); - if (0 == numOfRows || numOfRows <= existRows) { +static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo *pBlockInfo, uint32_t numOfRows) { + ASSERT(numOfRows > 0 && pBlockInfo->capacity >= pBlockInfo->rows); + if (numOfRows < pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } + int32_t existedRows = pBlockInfo->rows; + if (IS_VAR_DATA_TYPE(pColumn->info.type)) { char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); - if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } pColumn->varmeta.offset = (int32_t*)tmp; - memset(&pColumn->varmeta.offset[existRows], 0, sizeof(int32_t) * (numOfRows - existRows)); + memset(&pColumn->varmeta.offset[existedRows], 0, sizeof(int32_t) * (numOfRows - existedRows)); } else { char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - int32_t oldLen = BitmapLen(existRows); + int32_t oldLen = BitmapLen(existedRows); pColumn->nullbitmap = tmp; memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); @@ -1160,8 +1166,8 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, ui if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - memset(tmp + pColumn->info.bytes * existRows, 0, pColumn->info.bytes * (numOfRows - existRows)); - + + memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); pColumn->pData = tmp; } @@ -1178,6 +1184,11 @@ void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { } } +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { + SDataBlockInfo info = {0}; + return doEnsureCapacity(pColumn, &info, numOfRows); +} + int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; ASSERT(numOfRows > 0); @@ -1188,9 +1199,10 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { pDataBlock->info.capacity = numOfRows; - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - code = colInfoDataEnsureCapacity(p, pDataBlock->info.rows, numOfRows); + code = doEnsureCapacity(p, &pDataBlock->info, numOfRows); if (code) { return code; } @@ -1214,40 +1226,34 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { return NULL; } - int32_t numOfCols = pDataBlock->info.numOfCols; - - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - - pBlock->info.numOfCols = numOfCols; - pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; - pBlock->info.rowSize = pDataBlock->info.rowSize; - pBlock->info.groupId = pDataBlock->info.groupId; - pBlock->info.childId = pDataBlock->info.childId; - pBlock->info.type = pDataBlock->info.type; + SSDataBlock* pBlock = createDataBlock(); + pBlock->info = pDataBlock->info; + pBlock->info.rows = 0; + pBlock->info.capacity = 0; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colInfo = {0}; SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - colInfo.info = p->info; - colInfo.hasNull = true; - taosArrayPush(pBlock->pDataBlock, &colInfo); + SColumnInfoData colInfo = {.hasNull = true, .info = p->info}; + blockDataAppendColInfo(pBlock, &colInfo); } if (copyData) { + int32_t code = blockDataEnsureCapacity(pBlock, pDataBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + blockDataDestroy(pBlock); + return NULL; + } + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); - - int32_t code = colInfoDataEnsureCapacity(pDst, 0, pDataBlock->info.rows); - if (code != TSDB_CODE_SUCCESS) { - return NULL; - } - if (pSrc->pData == NULL) { continue; } - colDataAssign(pDst, pSrc, pDataBlock->info.rows); + + colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); } pBlock->info.rows = pDataBlock->info.rows; @@ -1257,16 +1263,74 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { return pBlock; } +SSDataBlock* createDataBlock() { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + if (pBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + if (pBlock->pDataBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pBlock); + } + + return pBlock; +} + +int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData) { + ASSERT(pBlock != NULL && pColInfoData != NULL); + if (pBlock->pDataBlock == NULL) { + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + if (pBlock->pDataBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + } + + void* p = taosArrayPush(pBlock->pDataBlock, pColInfoData); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + ASSERT(pColInfoData->info.type != 0); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + pBlock->info.hasVarCol = true; + } + + pBlock->info.rowSize += pColInfoData->info.bytes; + return TSDB_CODE_SUCCESS; +} + +SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) { + SColumnInfoData col = {.hasNull = true}; + col.info.colId = colId; + col.info.type = type; + col.info.bytes = bytes; + + return col; +} + +SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index) { + ASSERT(pBlock != NULL); + if (index >= taosArrayGetSize(pBlock->pDataBlock)) { + return NULL; + } + + return taosArrayGet(pBlock->pDataBlock, index); +} + size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { - int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock->info.numOfCols); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols); int32_t rowSize = pBlock->info.rowSize; - int32_t nRows = payloadSize / rowSize; // the true value must be less than the value of nRows int32_t additional = 0; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pCol->info.type)) { additional += nRows * sizeof(int32_t); @@ -1350,7 +1414,8 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { if (pBlock->info.rows <= n) { blockDataCleanup(pBlock); } else { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows); } @@ -1362,7 +1427,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int64_t tbUid = pBlock->info.uid; - int16_t numOfCols = pBlock->info.numOfCols; + int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int16_t hasVarCol = pBlock->info.hasVarCol; int32_t rows = pBlock->info.rows; int32_t sz = taosArrayGetSize(pBlock->pDataBlock); @@ -1397,8 +1462,10 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { int32_t sz; - buf = taosDecodeFixedI64(buf, &pBlock->info.uid); - buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols); + int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + + buf = taosDecodeFixedU64(buf, &pBlock->info.uid); + buf = taosDecodeFixedI16(buf, &numOfCols); buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol); buf = taosDecodeFixedI32(buf, &pBlock->info.rows); buf = taosDecodeFixedI32(buf, &sz); @@ -1507,12 +1574,13 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); for (int32_t j = 0; j < rows; j++) { printf("%s |", flag); - for (int32_t k = 0; k < colNum; k++) { + for (int32_t k = 0; k < numOfCols; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); if (colDataIsNull(pColInfoData, rows, j, NULL)) { @@ -1565,7 +1633,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks int32_t bufSize = sizeof(SSubmitReq); for (int32_t i = 0; i < sz; ++i) { SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info; - bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols)); + + int32_t numOfCols = taosArrayGetSize(pDataBlocks); + bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(numOfCols)); bufSize += sizeof(SSubmitBlk); } @@ -1583,10 +1653,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks for (int32_t i = 0; i < sz; ++i) { SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; - int32_t rowSize = pDataBlock->info.rowSize; - int64_t groupId = pDataBlock->info.groupId; +// int32_t rowSize = pDataBlock->info.rowSize; +// int64_t groupId = pDataBlock->info.groupId; if (colNum <= 1) { // invalid if only with TS col @@ -1819,9 +1889,6 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t taosArraySetSize(pBlock->pDataBlock, numOfCols); } - pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock); - ASSERT(pBlock->pDataBlock->size >= numOfCols); - for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData->info.type = *(int16_t*)pStart; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 2adb558d01..8460d71e56 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -88,26 +88,15 @@ TEST(testCase, toInteger_test) { } TEST(testCase, Datablock_test) { - SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - b->info.numOfCols = 2; - b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SSDataBlock* b = createDataBlock(); - SColumnInfoData infoData = {0}; - infoData.info.bytes = 4; - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.colId = 1; - - infoData.pData = (char*)taosMemoryCalloc(40, infoData.info.bytes); - infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (40 / 8)); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1); taosArrayPush(b->pDataBlock, &infoData); + blockDataAppendColInfo(b, &infoData); - SColumnInfoData infoData1 = {0}; - infoData1.info.bytes = 40; - infoData1.info.type = TSDB_DATA_TYPE_BINARY; - infoData1.info.colId = 2; - - infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(40, sizeof(uint32_t)); - taosArrayPush(b->pDataBlock, &infoData1); + SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2); + blockDataAppendColInfo(b, &infoData1); + blockDataEnsureCapacity(b, 40); char* str = "the value of: %d"; char buf[128] = {0}; @@ -222,28 +211,17 @@ TEST(testCase, non_var_dataBlock_split_test) { #endif TEST(testCase, var_dataBlock_split_test) { - SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - b->info.numOfCols = 2; - b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - int32_t numOfRows = 1000000; - SColumnInfoData infoData = {0}; - infoData.info.bytes = 4; - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.colId = 1; + SSDataBlock* b = createDataBlock(); - infoData.pData = (char*)taosMemoryCalloc(numOfRows, infoData.info.bytes); - infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (numOfRows / 8)); - taosArrayPush(b->pDataBlock, &infoData); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1); + blockDataAppendColInfo(b, &infoData); - SColumnInfoData infoData1 = {0}; - infoData1.info.bytes = 40; - infoData1.info.type = TSDB_DATA_TYPE_BINARY; - infoData1.info.colId = 2; + SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2); + blockDataAppendColInfo(b, &infoData1); - infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(numOfRows, sizeof(uint32_t)); - taosArrayPush(b->pDataBlock, &infoData1); + blockDataEnsureCapacity(b, numOfRows); char buf[41] = {0}; char buf1[100] = {0}; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index fed6a5a721..27de3883e9 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -236,7 +236,6 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {0}; @@ -271,7 +270,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + - blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols); + blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 70d89102e6..b58450ad44 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -149,8 +149,8 @@ int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidL int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, - int32_t *pNumOfRows, int16_t *pNumOfCols); +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, + int32_t *pNumOfRows); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 7c75d88a83..e1c3322a04 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs // TODO enable compress int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); @@ -74,7 +74,6 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR if (pDataBlock == NULL) break; ASSERT(pDataBlock->info.rows != 0); - ASSERT(pDataBlock->info.numOfCols != 0); tqAddBlockDataToRsp(pDataBlock, pRsp); if (pRsp->withTbName) { @@ -88,8 +87,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, - &block.info.numOfCols) < 0) { + if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } @@ -106,8 +104,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, - &block.info.numOfCols) < 0) { + if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 0c38d6442b..fa0eb1d458 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -123,8 +123,8 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) { return false; } -int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, - int32_t* pNumOfRows, int16_t* pNumOfCols) { +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, + int32_t* pNumOfRows) { *pUid = 0; // TODO set to real sversion @@ -161,23 +161,14 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); if (colNumNeed == 0) { - *ppCols = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData)); - if (*ppCols == NULL) { - return -1; - } - int32_t colMeta = 0; while (colMeta < pSchemaWrapper->nCols) { SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - SColumnInfoData colInfo = {0}; - colInfo.info.bytes = pColSchema->bytes; - colInfo.info.colId = pColSchema->colId; - colInfo.info.type = pColSchema->type; - - if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { goto FAIL; } - taosArrayPush(*ppCols, &colInfo); colMeta++; } } else { @@ -185,11 +176,6 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p colNumNeed = pSchemaWrapper->nCols; } - *ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); - if (*ppCols == NULL) { - return -1; - } - int32_t colMeta = 0; int32_t colNeed = 0; while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { @@ -201,23 +187,22 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p } else if (colIdSchema > colIdNeed) { colNeed++; } else { - SColumnInfoData colInfo = {0}; - colInfo.info.bytes = pColSchema->bytes; - colInfo.info.colId = pColSchema->colId; - colInfo.info.type = pColSchema->type; - - if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { goto FAIL; } - taosArrayPush(*ppCols, &colInfo); colMeta++; colNeed++; } } } - int32_t colActual = taosArrayGetSize(*ppCols); - *pNumOfCols = colActual; + if (blockDataEnsureCapacity(pBlock, *pNumOfRows) < 0) { + goto FAIL; + } + + int32_t colActual = blockDataGetNumOfCols(pBlock); // TODO in stream shuffle case, fetch groupId *pGroupId = 0; @@ -234,7 +219,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p tdSTSRowIterReset(&iter, row); // get all wanted col of that block for (int32_t i = 0; i < colActual; i++) { - SColumnInfoData* pColData = taosArrayGet(*ppCols, i); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SCellVal sVal = {0}; if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; @@ -246,8 +231,9 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p curRow++; } return 0; -FAIL: - if (*ppCols) taosArrayDestroy(*ppCols); + +FAIL: // todo refactor here +// if (*ppCols) taosArrayDestroy(*ppCols); return -1; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b628f0dde5..393cb9f749 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -44,7 +44,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.ctb.suid = suid; STagVal tagVal = { - .cid = pDataBlock->info.numOfCols + 1, + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = (int64_t)pDataBlock->info.groupId, }; @@ -109,7 +109,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.ctb.suid = suid; STagVal tagVal = { - .cid = pDataBlock->info.numOfCols + 1, + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = (int64_t)pDataBlock->info.groupId, }; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ce73246e51..1a623f1978 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -16,13 +16,11 @@ #include "tsdb.h" #include "vnode.h" -#define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) -#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) +#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pResBlock->pDataBlock))) #define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ ((SDataBlockInfo){.window = {.skey = (_block)->minKey.ts, .ekey = (_block)->maxKey.ts}, \ - .numOfCols = (_block)->numOfCols, \ .rows = (_block)->numOfRows, \ .uid = (_checkInfo)->tableId}) @@ -115,7 +113,8 @@ typedef struct STsdbReadHandle { // SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time // SColumnDataAgg** pstatis;// the ptr array list to return to caller int32_t numOfBlocks; - SArray* pColumns; // column list, SColumnInfoData array list + SSDataBlock* pResBlock; +// SArray* pColumns; // column list, SColumnInfoData array list bool locateStart; int32_t outputCapacity; int32_t realNumOfRows; @@ -177,7 +176,7 @@ static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) { SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); taosArrayPush(pIdList, &pCol->info.colId); } @@ -425,23 +424,21 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* } // todo: use list instead of array? - pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); - if (pReadHandle->pColumns == NULL) { + pReadHandle->pResBlock = createDataBlock(); + if (pReadHandle->pResBlock == NULL) { goto _end; } for (int32_t i = 0; i < pCond->numOfCols; ++i) { - SColumnInfoData colInfo = {{0}, 0}; - colInfo.info = pCond->colList[i]; - - int32_t code = colInfoDataEnsureCapacity(&colInfo, 0, pReadHandle->outputCapacity); - if (code != TSDB_CODE_SUCCESS) { + SColumnInfoData colInfo = {.info = pCond->colList[i], 0}; + int32_t code = blockDataAppendColInfo(pReadHandle->pResBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS){ goto _end; } - - taosArrayPush(pReadHandle->pColumns, &colInfo); } + blockDataEnsureCapacity(pReadHandle->pResBlock, pReadHandle->outputCapacity); + pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); size_t size = taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn); @@ -1536,12 +1533,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t int32_t trueEnd = ascScan ? end : start; int32_t step = ascScan ? 1 : -1; - int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); // data in buffer has greater timestamp, copy data in file block int32_t i = 0, j = 0; while (i < requiredNumOfCols && j < pCols->numOfCols) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); SDataCol* src = &pCols->cols[j]; if (src->colId < pColInfo->info.colId) { @@ -1592,7 +1589,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t } while (i < requiredNumOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colDataAppendNNULL(pColInfo, numOfRows, num); i++; } @@ -1674,7 +1671,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa int32_t i = 0, j = 0, k = 0; while (i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); int32_t colIdOfRow1; if (j >= numOfColsOfRow1) { @@ -1805,7 +1802,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (mergeOption == 1) { while (i < numOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colDataAppend(pColInfo, *curRow, NULL, true); ++i; } @@ -1861,7 +1858,7 @@ static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) { assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey); } - SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0); assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows - 1]); } else { @@ -2771,7 +2768,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; int curRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); win->skey = TSKEY_INITIAL_VAL; @@ -2901,7 +2898,7 @@ static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) { if (exists) { tsdbRetrieveDataBlock((tsdbReaderT*)pTsdbReadHandle, NULL); if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0); assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); } @@ -3003,7 +3000,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // // int32_t i = 0, j = 0; // while(i < tgNumOfCols && j < numOfCols) { -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // if (pTable->lastCols[j].colId < pColInfo->info.colId) { // j++; // continue; @@ -3066,7 +3063,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // continue; // } // -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // // if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -3091,7 +3088,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // // // leave the real ts column as the last row, because last function only (not stable) use the last row as res // if (priKey != TSKEY_INITIAL_VAL) { -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, priIdx); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; // // *(TSKEY *)pData = priKey; @@ -3101,7 +3098,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // continue; // } // -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // // assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID); @@ -3161,9 +3158,9 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { bool tsdbNextDataBlock(tsdbReaderT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)pHandle; - size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pColumns); + size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colInfoDataCleanup(pColInfo, pTsdbReadHandle->outputCapacity); } @@ -3250,7 +3247,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next; // // for (int32_t i = 0; i < numOfCols; ++i) { -// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); +// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // // SColumnInfoData colInfo = {{0}, 0}; // colInfo.info = pCol->info; @@ -3280,7 +3277,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // } // // for (int32_t i = 0; i < cond.numOfCols; ++i) { -// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); +// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); // } // @@ -3342,7 +3339,7 @@ bool tsdbGetExternalRow(tsdbReaderT pHandle) { int32_t numOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i); memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); @@ -3617,13 +3614,13 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { */ STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; if (pHandle->cur.fid == INT32_MIN) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; if (pHandle->cur.mixBlock) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); assert(pHandle->realNumOfRows <= binfo.rows); @@ -3633,7 +3630,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && pBlockLoadInfo->uid == pCheckInfo->tableId) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { // only load the file block SBlock* pBlock = pBlockInfo->compBlock; if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) { @@ -3641,7 +3638,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { } int32_t numOfRows = doCopyRowsFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1); - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } } } @@ -3692,7 +3689,7 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { return; } - pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); + pTsdbReadHandle->pResBlock->pDataBlock = doFreeColumnInfoData(pTsdbReadHandle->pResBlock->pDataBlock); taosArrayDestroy(pTsdbReadHandle->suppInfo.defaultLoadColumn); taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo); diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 2c1e6fbbbd..2a8c7a583c 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -435,7 +435,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { SSDataBlock *pDataBlock = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); EXPECT_NE(pDataBlock, nullptr); pDataBlock->pBlockAgg = NULL; - pDataBlock->info.numOfCols = tSmaNumOfCols; + taosArrayGetSize(pDataBlock->pDataBlock) = tSmaNumOfCols; pDataBlock->info.rows = tSmaNumOfRows; pDataBlock->info.groupId = tSmaGroupId + g; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index c6a0e52a42..66dc2bcaaf 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -30,31 +30,19 @@ static int32_t getSchemaBytes(const SSchema* pSchema) { } static SSDataBlock* buildDescResultDataBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = DESCRIBE_RESULT_COLS; - pBlock->info.hasVarCol = true; + SSDataBlock* pBlock = createDataBlock(); - pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_FIELD_LEN, 1); + blockDataAppendColInfo(pBlock, &infoData); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_FIELD_LEN; + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_TYPE_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; - - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_NOTE_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, tDataTypes[TSDB_DATA_TYPE_INT].bytes, 3); + blockDataAppendColInfo(pBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_NOTE_LEN, 4); + blockDataAppendColInfo(pBlock, &infoData); return pBlock; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index ae0f669d65..091f300428 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1190,17 +1190,12 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE; - - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - taosArrayPush(pBlock->pDataBlock, &infoData); + SSDataBlock *pBlock = createDataBlock(); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1); + blockDataAppendColInfo(pBlock, &infoData); + blockDataEnsureCapacity(pBlock, rowNum); SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); - pInfoData->hasNull = false; - colInfoDataEnsureCapacity(pInfoData, 0, rowNum); char buf[1024] = {0}; for (int32_t i = 0; i < rowNum; ++i) { @@ -1210,9 +1205,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { colDataAppend(pInfoData, i, buf, false); } - pBlock->info.numOfCols = 1; pBlock->info.rows = rowNum; - pBlock->info.hasVarCol = true; int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); @@ -1226,7 +1219,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htonl(rowNum); int32_t len = 0; - blockCompressEncode(pBlock, rsp->data, &len, pBlock->info.numOfCols, 0); + blockCompressEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 33b7811e6c..5ba275c7b2 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -75,7 +75,7 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = 0; pEntry->numOfRows = pInput->pData->info.rows; - pEntry->numOfCols = pInput->pData->info.numOfCols; + pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock); pEntry->dataLen = sizeof(SDeleterRes); ASSERT(1 == pEntry->numOfRows); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 566abb1451..60560e49dd 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -183,34 +183,24 @@ SArray* createSortInfo(SNodeList* pNodeList) { SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { int32_t numOfCols = LIST_LENGTH(pNode->pSlots); - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + SSDataBlock* pBlock = createDataBlock(); pBlock->info.blockId = pNode->dataBlockId; - pBlock->info.rowSize = pNode->totalRowSize; // todo ?? pBlock->info.type = STREAM_INVALID; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = {{0}}; SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); // if (!pDescNode->output) { // todo disable it temporarily // continue; // } - idata.info.type = pDescNode->dataType.type; - idata.info.bytes = pDescNode->dataType.bytes; + SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); idata.info.scale = pDescNode->dataType.scale; - idata.info.slotId = pDescNode->slotId; idata.info.precision = pDescNode->dataType.precision; - if (IS_VAR_DATA_TYPE(idata.info.type)) { - pBlock->info.hasVarCol = true; - } - - taosArrayPush(pBlock->pDataBlock, &idata); + blockDataAppendColInfo(pBlock, &idata); } - pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock); return pBlock; } @@ -600,7 +590,7 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray if (p->info.colId == pmInfo->colId) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, p, pBlock->info.rows); + colDataAssign(pDst, p, pBlock->info.rows, &pBlock->info); i++; j++; } else if (p->info.colId < pmInfo->colId) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index cf72dfd796..e805de3727 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -68,6 +68,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayClear(p->pDataBlock); taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock); + printf("size------------->%ld, %ld, %p, \n", taosArrayGetSize(pDataBlock->pDataBlock), taosArrayGetSize(pDataBlock->pDataBlock) * pDataBlock->pDataBlock->elemSize, + pDataBlock->pDataBlock); taosArrayPush(pInfo->pBlockLists, &p); } } else { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 20dac85223..72c480a81b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -356,7 +356,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; pColData->info.bytes = sizeof(int64_t); - colInfoDataEnsureCapacity(pColData, 0, 5); + colInfoDataEnsureCapacity(pColData, 5); colDataAppendInt64(pColData, 0, &pQueryWindow->skey); colDataAppendInt64(pColData, 1, &pQueryWindow->ekey); @@ -458,7 +458,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc pColInfo = pInput->pData[paramIndex]; } - colInfoDataEnsureCapacity(pColInfo, 0, numOfRows); + colInfoDataEnsureCapacity(pColInfo, numOfRows); int8_t type = pFuncParam->param.nType; if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { @@ -579,7 +579,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0], pfCtx->input.numOfRows); } else { - colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); + colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows, &pResult->info); } numOfRows = pfCtx->input.numOfRows; @@ -1186,7 +1186,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc // } // if (pQueryAttr->pFilters != NULL) { -// filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); +// filterSetColFieldData(pQueryAttr->pFilters, taosArrayGetSize(pBlock->pDataBlock), pBlock->pDataBlock); // } // if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { @@ -1321,7 +1321,8 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { // todo move to the initialization function int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); - SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock}; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + SFilterColumnParam param1 = {.numOfCols = numOfCols, .pDataBlock = pBlock->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); int8_t* rowRes = NULL; @@ -1343,7 +1344,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR int32_t totalRows = pBlock->info.rows; SSDataBlock* px = createOneDataBlock(pBlock, true); - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); // it is a reserved column for scalar function, and no data in this column yet. @@ -2064,29 +2066,20 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo pStart += sizeof(SSysTableSchema); } - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; - + SSDataBlock* pBlock = createDataBlock(); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = {0}; - - idata.info.type = pSchema[i].type; - idata.info.bytes = pSchema[i].bytes; - idata.info.colId = pSchema[i].colId; - idata.hasNull = true; - - taosArrayPush(pBlock->pDataBlock, &idata); - if (IS_VAR_DATA_TYPE(idata.info.type)) { - pBlock->info.hasVarCol = true; - } + SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); + blockDataAppendColInfo(pBlock, &idata); } blockCompressDecode(pBlock, numOfCols, numOfRows, pStart); + blockDataEnsureCapacity(pRes, numOfRows); + + relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); // data from mnode pRes->info.rows = numOfRows; - relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); + taosArrayDestroy(pBlock->pDataBlock); taosMemoryFree(pBlock); // blockDataDestroy(pBlock); @@ -2412,15 +2405,15 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode tsem_init(&pInfo->ready, 0, 0); - pInfo->seqLoadData = false; + pInfo->seqLoadData = false; pInfo->pTransporter = pTransporter; - pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); - pOperator->name = "ExchangeOperator"; + pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pInfo->pResult->info.numOfCols; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pResult->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, @@ -2542,9 +2535,6 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock SSortedMergeOperatorInfo* pInfo = pOperator->info; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - // pCtx[i].size = 1; - } for (int32_t i = 0; i < pBlock->info.rows; ++i) { if (!pInfo->hasGroupVal) { @@ -2653,7 +2643,7 @@ SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 064a36df1c..4e4aaba7f4 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -559,8 +559,8 @@ uint64_t calcGroupId(char* pData, int32_t len) { } int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { - size_t numOfCols = pBlock->info.numOfCols; - int32_t* offset = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(int32_t)); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t)); offset[0] = sizeof(int32_t) + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index eed357c86d..8c6797759e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -203,7 +203,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pCost->skipBlocks += 1; // clear all data in pBlock that are set when handing the previous block - for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); pcol->pData = NULL; } @@ -217,7 +217,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); if (allColumnsHaveAgg == true) { - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); // todo create this buffer during creating operator if (pBlock->pBlockAgg == NULL) { @@ -316,7 +316,7 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_ SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); - colInfoDataEnsureCapacity(pColInfoData, 0, pBlock->info.rows); + colInfoDataEnsureCapacity(pColInfoData, pBlock->info.rows); colInfoDataCleanup(pColInfoData, pBlock->info.rows); int32_t functionId = pExpr->pExpr->_function.functionId; @@ -354,10 +354,8 @@ void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* p struct SScalarFuncExecFuncs fpSet = {0}; fmGetScalarFuncExecFuncs(functionId, &fpSet); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_BIGINT; - infoData.info.bytes = sizeof(uint64_t); - colInfoDataEnsureCapacity(&infoData, 0, 1); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_BIGINT, sizeof(uint64_t), 1); + colInfoDataEnsureCapacity(&infoData, 1); colDataAppendInt64(&infoData, 0, (int64_t*)&pBlock->info.uid); SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData}; @@ -785,7 +783,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { } static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) { - for (int32_t j = 0; j < source->info.numOfCols; j++) { + for (int32_t j = 0; j < taosArrayGetSize(source->pDataBlock); j++) { SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j); SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j); if (colDataIsNull_s(pSourceCol, sourceRowId)) { @@ -857,7 +855,6 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); blockDataEnsureCapacity(pUpdateBlock, size); - ASSERT(pBlock->info.numOfCols == pUpdateBlock->info.numOfCols); int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex); pInfo->groupId = getGroupId(pInfo->pOperatorDumy, pBlock, rowId); @@ -968,19 +965,20 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); while (tqNextDataBlock(pInfo->streamBlockReader)) { - SArray* pCols = NULL; + SSDataBlock block = {0}; uint64_t groupId = 0; uint64_t uid = 0; int32_t numOfRows = 0; - int16_t outputCol = 0; - int32_t code = tqRetrieveDataBlock(&pCols, pInfo->streamBlockReader, &groupId, &uid, &numOfRows, &outputCol); + int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows); if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { pTaskInfo->code = code; return NULL; } + blockDataEnsureCapacity(pInfo->pRes, numOfRows); + pInfo->pRes->info.groupId = groupId; pInfo->pRes->info.rows = numOfRows; pInfo->pRes->info.uid = uid; @@ -999,6 +997,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->pRes->info.groupId = *groupIdPre; } + // todo extract method for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); if (!pColMatchInfo->output) { @@ -1006,8 +1005,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } bool colExists = false; - for (int32_t j = 0; j < taosArrayGetSize(pCols); ++j) { - SColumnInfoData* pResCol = taosArrayGet(pCols, j); + for (int32_t j = 0; j < blockDataGetNumOfCols(&block); ++j) { + SColumnInfoData* pResCol = bdGetColumnInfoData(&block, j); if (pResCol->info.colId == pColMatchInfo->colId) { taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); colExists = true; @@ -1018,7 +1017,6 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { // the required column does not exists in submit block, let's set it to be all null value if (!colExists) { SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); - colInfoDataEnsureCapacity(pDst, 0, pBlockInfo->rows); colDataAppendNNULL(pDst, 0, pBlockInfo->rows); } } @@ -1164,7 +1162,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pInfo->pRes->info.numOfCols; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = @@ -1292,7 +1290,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i); if (keep) { - colDataAssign(pDest, pSrc, pInfo->pRes->info.rows); + colDataAssign(pDest, pSrc, pInfo->pRes->info.rows, &px->info); numOfRow = pInfo->pRes->info.rows; } else if (NULL != rowRes) { numOfRow = 0; @@ -1322,8 +1320,6 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { } static SSDataBlock* buildSysTableMetaBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - size_t size = 0; const SSysTableMeta* pMeta = NULL; getInfosDbMeta(&pMeta, &size); @@ -1336,19 +1332,12 @@ static SSDataBlock* buildSysTableMetaBlock() { } } - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - + SSDataBlock* pBlock = createDataBlock(); for (int32_t i = 0; i < pMeta[index].colNum; ++i) { - SColumnInfoData colInfoData = {0}; - colInfoData.info.colId = i + 1; - colInfoData.info.type = pMeta[index].schema[i].type; - colInfoData.info.bytes = pMeta[index].schema[i].bytes; - taosArrayPush(pBlock->pDataBlock, &colInfoData); + SColumnInfoData colInfoData = createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); + blockDataAppendColInfo(pBlock, &colInfoData); } - pBlock->info.numOfCols = pMeta[index].colNum; - pBlock->info.hasVarCol = true; - return pBlock; } @@ -1696,7 +1685,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pResBlock->info.numOfCols; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = @@ -2033,7 +2022,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc pCost->skipBlocks += 1; // clear all data in pBlock that are set when handing the previous block - for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); pcol->pData = NULL; } @@ -2048,7 +2037,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc tsdbRetrieveDataBlockStatisInfo(reader, &pColAgg, &allColumnsHaveAgg); if (allColumnsHaveAgg == true) { - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); // todo create this buffer during creating operator if (pBlock->pBlockAgg == NULL) { diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 9d13276e6d..feb81b1708 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -78,7 +78,7 @@ _error: } void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); bool isNull = tsortIsNullVal(pTupleHandle, i); if (isNull) { @@ -118,6 +118,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i } if (p->info.rows > 0) { + // todo extract function to handle this int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); @@ -125,7 +126,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; @@ -317,7 +318,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 110c2d6a8f..1d82d6a644 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -50,7 +50,7 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) { // the first are always the timestamp column, so start from the second column. - for (int32_t i = 1; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i); colDataAppendNULL(p, rowIndex); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index bde4aa18e8..2dbdbd7069 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1821,7 +1821,7 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB return TSDB_CODE_OUT_OF_MEMORY; } - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); @@ -2234,9 +2234,6 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) } static void clearUpdateDataBlock(SSDataBlock* pBlock) { - if (pBlock->info.rows <= 0) { - return; - } blockDataCleanup(pBlock); } @@ -2245,12 +2242,14 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol clearUpdateDataBlock(pDest); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex); + // copy timestamp column - colDataAssign(pDestCol, pSourceCol, pSource->info.rows); - for (int32_t i = 1; i < pDest->info.numOfCols; i++) { + colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info); + for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) { SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i); colDataAppendNNULL(pCol, 0, pSource->info.rows); } + pDest->info.rows = pSource->info.rows; pDest->info.groupId = pSource->info.groupId; pDest->info.type = pSource->info.type; @@ -2919,7 +2918,7 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false); - for (int32_t i = 1; i < pBlock->info.numOfCols; i++) { + for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); i++) { pColInfoData = taosArrayGet(pBlock->pDataBlock, i); colDataAppendNULL(pColInfoData, pBlock->info.rows); } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index f21cad2dd6..caf567884f 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -146,7 +146,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock); // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation - int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock->info.numOfCols))/rowSize; + int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)))/rowSize; ASSERT(numOfRows > 0); return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); } @@ -177,7 +177,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { return terrno; } - int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t); + int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t); assert(size <= getBufPageSize(pHandle->pBuf)); blockDataToBuf(pPage, p); @@ -251,7 +251,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int } static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i); @@ -498,7 +498,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { return terrno; } - int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t); + int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t); assert(size <= getBufPageSize(pHandle->pBuf)); blockDataToBuf(pPage, pDataBlock); diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index dba2fb897a..3c12889aca 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -62,18 +62,11 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = createDataBlock(); - pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); - - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pInfo->pBlock, &colInfo); + blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage); // SColumnInfoData colInfo1 = {0}; // colInfo1.info.type = TSDB_DATA_TYPE_BINARY; @@ -115,7 +108,6 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { } pBlock->info.rows = pInfo->numOfRowsPerPage; - pBlock->info.numOfCols = 1; pInfo->current += 1; return pBlock; @@ -128,28 +120,15 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = createDataBlock(); - pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1); + blockDataAppendColInfo(pInfo->pBlock, &colInfo); - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP; - colInfo.info.bytes = sizeof(int64_t); - colInfo.info.colId = 1; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); - // colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + SColumnInfoData colInfo1 = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2); + blockDataAppendColInfo(pInfo->pBlock, &colInfo1); - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); - - SColumnInfoData colInfo1 = {0}; - colInfo1.info.type = TSDB_DATA_TYPE_INT; - colInfo1.info.bytes = 4; - colInfo1.info.colId = 2; - - colInfo1.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo1.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); - - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); + blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage); } else { blockDataCleanup(pInfo->pBlock); } @@ -185,7 +164,6 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { } pBlock->info.rows = pInfo->numOfRowsPerPage; - pBlock->info.numOfCols = 1; pInfo->current += 1; diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index 66ed078bbe..70def78263 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -62,25 +62,21 @@ SSDataBlock* getSingleColDummyBlock(void* param) { return NULL; } - SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SSDataBlock* pBlock = createDataBlock(); SColumnInfoData colInfo = {0}; colInfo.info.type = pInfo->type; if (pInfo->type == TSDB_DATA_TYPE_NCHAR){ colInfo.info.bytes = TSDB_NCHAR_SIZE * VARCOUNT + VARSTR_HEADER_SIZE; - colInfo.varmeta.offset = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); } else if(pInfo->type == TSDB_DATA_TYPE_BINARY) { colInfo.info.bytes = VARCOUNT + VARSTR_HEADER_SIZE; - colInfo.varmeta.offset = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); } else{ colInfo.info.bytes = tDataTypes[pInfo->type].bytes; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->pageRows, colInfo.info.bytes)); - colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8)); } colInfo.info.colId = 1; - taosArrayPush(pBlock->pDataBlock, &colInfo); + blockDataAppendColInfo(pBlock, &colInfo); + blockDataEnsureCapacity(pBlock, pInfo->pageRows); for (int32_t i = 0; i < pInfo->pageRows; ++i) { SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); @@ -128,7 +124,6 @@ SSDataBlock* getSingleColDummyBlock(void* param) { } pBlock->info.rows = pInfo->pageRows; - pBlock->info.numOfCols = 1; return pBlock; } @@ -354,15 +349,10 @@ TEST(testCase, ordered_merge_sort_Test) { SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); taosArrayPush(orderInfo, &oi); - SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - taosArrayPush(pBlock->pDataBlock, &colInfo); + SSDataBlock* pBlock = createDataBlock(); + for (int32_t i = 0; i < 1; ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); } SSortHandle* phandle = tsortCreateSortHandle(orderInfo, SORT_MULTISOURCE_MERGE, 1024, 5, pBlock,"test_abc"); diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index c3d61d426d..669c2635b5 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -28,19 +28,6 @@ extern "C" { #include "function.h" #include "tudf.h" - -extern SAggFunctionInfo aggFunc[35]; - -#define FUNCSTATE_SO 0x0u -#define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM -#define FUNCSTATE_STREAM 0x2u // function avail for stream -#define FUNCSTATE_STABLE 0x4u // function avail for super table -#define FUNCSTATE_NEED_TS 0x8u // timestamp is required during query processing -#define FUNCSTATE_SELECTIVITY 0x10u // selectivity functions, can exists along with tag columns - -#define BASIC_FUNC_SO FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE -#define BASIC_FUNC_MO FUNCSTATE_MO | FUNCSTATE_STREAM | FUNCSTATE_STABLE - #define AVG_FUNCTION_INTER_BUFFER_SIZE 50 #define DATA_SET_FLAG ',' // to denote the output area has data, not null value diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 076ae61460..784017fb05 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1427,11 +1427,13 @@ void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) { int32_t pageId = pTuplePos->pageId; int32_t offset = pTuplePos->offset; + + int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock); if (pTuplePos->pageId != -1) { SFilePage* pPage = getBufPage(pCtx->pBuf, pageId); bool* nullList = (bool*)((char*)pPage + offset); - char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool)); + char* pStart = (char*)(nullList + numOfCols * sizeof(bool)); // todo set the offset value to optimize the performance. for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { @@ -3018,7 +3020,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = NULL; - int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool); + int32_t completeRowSize = pSrcBlock->info.rowSize + (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock) * sizeof(bool); if (pCtx->curBufPage == -1) { pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage); @@ -3036,8 +3038,8 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS // keep the current row data, extract method int32_t offset = 0; bool* nullList = (bool*)((char*)pPage + pPage->num); - char* pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols); - for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + char* pStart = (char*)(nullList + sizeof(bool) * (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock)); + for (int32_t i = 0; i < (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock); ++i) { SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); bool isNull = colDataIsNull_s(pCol, rowIndex); if (isNull) { @@ -3066,11 +3068,13 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId); + int32_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); + bool* nullList = (bool*)((char*)pPage + pPos->offset); - char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool)); + char* pStart = (char*)(nullList + numOfCols * sizeof(bool)); int32_t offset = 0; - for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) { offset += pCol->info.bytes; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index f6ae027e48..0ecad278e3 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -763,8 +763,7 @@ void freeUdfInterBuf(SUdfInterBuf *buf) { int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) { udfBlock->numOfRows = block->info.rows; - udfBlock->numOfCols = block->info.numOfCols; - udfBlock->udfCols = taosMemoryCalloc(udfBlock->numOfCols, sizeof(SUdfColumn*)); + udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn*)); for (int32_t i = 0; i < udfBlock->numOfCols; ++i) { udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn)); SColumnInfoData *col= (SColumnInfoData*)taosArrayGet(block->pDataBlock, i); @@ -799,7 +798,6 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo } int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { - block->info.numOfCols = 1; block->info.rows = udfCol->colData.numOfRows; block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type); @@ -830,25 +828,19 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) { output->info.rows = input->numOfRows; - output->info.numOfCols = numOfCols; - bool hasVarCol = false; - for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) { - hasVarCol = true; - break; - } - } - output->info.hasVarCol = hasVarCol; - output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfCols; ++i) { taosArrayPush(output->pDataBlock, (input + i)->columnData); + + if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) { + output->info.hasVarCol = true; + } } return 0; } int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { - if (input->info.numOfCols != 1) { + if (taosArrayGetSize(input->pDataBlock) != 1) { fnError("scalar function only support one column"); return -1; } @@ -1090,23 +1082,14 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t numOfRows = pInput->numOfRows; - SSDataBlock tempBlock = {0}; - tempBlock.info.numOfCols = numOfCols; - tempBlock.info.rows = pInput->totalRows; - tempBlock.info.uid = pInput->uid; - bool hasVarCol = false; - tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - + SSDataBlock* pTempBlock = createDataBlock(); + pTempBlock->info.rows = pInput->totalRows; + pTempBlock->info.uid = pInput->uid; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData *col = pInput->pData[i]; - if (IS_VAR_DATA_TYPE(col->info.type)) { - hasVarCol = true; - } - taosArrayPush(tempBlock.pDataBlock, col); + blockDataAppendColInfo(pTempBlock, pInput->pData[i]); } - tempBlock.info.hasVarCol = hasVarCol; - SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows); + SSDataBlock *inputBlock = blockDataExtractBlock(pTempBlock, start, numOfRows); SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = session->bufSize, @@ -1131,7 +1114,9 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { } blockDataDestroy(inputBlock); - taosArrayDestroy(tempBlock.pDataBlock); + + taosArrayDestroy(pTempBlock->pDataBlock); + taosMemoryFree(pTempBlock); releaseUdfFuncHandle(pCtx->udfName); freeUdfInterBuf(&newState); diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index 15109db220..7395f1e14c 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -45,27 +45,27 @@ int scalarFuncTest() { for (int k = 0; k < 1; ++k) { SSDataBlock block = {0}; SSDataBlock *pBlock = █ - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; + for (int32_t i = 0; i < 1; ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); + } + + blockDataEnsureCapacity(pBlock, 1024); pBlock->info.rows = 1024; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows); - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(&colInfo, j, &j); - } - taosArrayPush(pBlock->pDataBlock, &colInfo); + + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, 0); + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(pCol, j, &j); } SScalarParam input = {0}; input.numOfRows = pBlock->info.rows; input.columnData = taosArrayGet(pBlock->pDataBlock, 0); + SScalarParam output = {0}; doCallUdfScalarFunc(handle, &input, 1, &output); taosArrayDestroy(pBlock->pDataBlock); + SColumnInfoData *col = output.columnData; for (int32_t i = 0; i < output.numOfRows; ++i) { if (i % 100 == 0) @@ -89,21 +89,18 @@ int aggregateFuncTest() { return -1; } - SSDataBlock block = {0}; - SSDataBlock *pBlock = █ - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; + SSDataBlock *pBlock = createDataBlock(); + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); + } + + blockDataEnsureCapacity(pBlock, 1024); pBlock->info.rows = 1024; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows); - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(&colInfo, j, &j); - } - taosArrayPush(pBlock->pDataBlock, &colInfo); + + SColumnInfoData* pColInfo = bdGetColumnInfoData(pBlock, 0); + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(pColInfo, j, &j); } SUdfInterBuf buf = {0}; @@ -121,6 +118,7 @@ int aggregateFuncTest() { freeUdfInterBuf(&resultBuf); doTeardownUdf(handle); + blockDataDestroy(pBlock); return 0; } diff --git a/source/libs/index/test/index_executor_tests.cpp b/source/libs/index/test/index_executor_tests.cpp index b88ffe5b8b..a0ac79a297 100644 --- a/source/libs/index/test/index_executor_tests.cpp +++ b/source/libs/index/test/index_executor_tests.cpp @@ -86,7 +86,7 @@ void sifAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *sl SColumnInfoData idata = {0}; idata.info = *colInfo; - colInfoDataEnsureCapacity(&idata, 0, rows); + colInfoDataEnsureCapacity(&idata, rows); taosArrayPush(res->pDataBlock, &idata); diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 4dd16098ff..c2d984c7dd 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -57,7 +57,7 @@ typedef struct SScalarCtx { #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); -SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); +SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); #define GET_PARAM_TYPE(_c) ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index a446aa7bdb..b8bcae161b 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3821,7 +3821,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData SScalarParam output = {0}; SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; - output.columnData = createColumnInfoData(&type, pSrc->info.rows); + output.columnData = sclCreateColumnInfoData(&type, pSrc->info.rows); SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 66e4af5ded..9890cff6f2 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -36,7 +36,7 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) { } -SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { +SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows) { SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); if (pColumnData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -48,7 +48,7 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { pColumnData->info.scale = pType->scale; pColumnData->info.precision = pType->precision; - int32_t code = colInfoDataEnsureCapacity(pColumnData, 0, numOfRows); + int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pColumnData); @@ -60,10 +60,10 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) { SScalarParam in = {.numOfRows = 1}; - in.columnData = createColumnInfoData(&pValueNode->node.resType, 1); + in.columnData = sclCreateColumnInfoData(&pValueNode->node.resType, 1); colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); - colInfoDataEnsureCapacity(out->columnData, 0, 1); + colInfoDataEnsureCapacity(out->columnData, 1); int32_t code = vectorConvertImpl(&in, out); sclFreeParam(&in); @@ -193,7 +193,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SValueNode *valueNode = (SValueNode *)node; param->numOfRows = 1; - param->columnData = createColumnInfoData(&valueNode->node.resType, 1); + param->columnData = sclCreateColumnInfoData(&valueNode->node.resType, 1); if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) { colDataAppendNULL(param->columnData, 0); } else { @@ -245,7 +245,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index); - if (NULL == block || ref->slotId >= block->info.numOfCols) { + if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) { sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -431,7 +431,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_JRET(code); } - output->columnData = createColumnInfoData(&node->node.resType, rowNum); + output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); if (output->columnData == NULL) { sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -484,7 +484,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o output->numOfRows = rowNum; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - output->columnData = createColumnInfoData(&t, rowNum); + output->columnData = sclCreateColumnInfoData(&t, rowNum); if (output->columnData == NULL) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -539,7 +539,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp int32_t code = 0; SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); - output->columnData = createColumnInfoData(&node->node.resType, rowNum); + output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); if (output->columnData == NULL) { sclError("calloc failed, size:%d", (int32_t)rowNum * node->node.resType.bytes); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -901,7 +901,7 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } - colDataAssign(col, res->columnData, res->numOfRows); + colDataAssign(col, res->columnData, res->numOfRows, NULL); block->info.rows = res->numOfRows; sclFreeParam(res); @@ -983,7 +983,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - colDataAssign(pDst->columnData, res->columnData, res->numOfRows); + colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); pDst->numOfRows = res->numOfRows; taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 2369779105..1c5a12687d 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -120,7 +120,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu } default: { - colDataAssign(pOutputData, pInputData, pInput->numOfRows); + colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL); } } @@ -273,7 +273,7 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP } default: { - colDataAssign(pOutputData, pInputData, pInput->numOfRows); + colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL); } } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index bff49c2ca9..a8a095a1cf 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -837,7 +837,7 @@ int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int3 SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; output->numOfRows = input->numOfRows; - output->columnData = createColumnInfoData(&t, input->numOfRows); + output->columnData = sclCreateColumnInfoData(&t, input->numOfRows); if (output->columnData == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -908,7 +908,7 @@ static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SSca pDest->numOfRows = pParam->numOfRows; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - pDest->columnData = createColumnInfoData(&t, pParam->numOfRows); + pDest->columnData = sclCreateColumnInfoData(&t, pParam->numOfRows); if (pDest->columnData == NULL) { sclError("malloc %d failed", (int32_t)(pParam->numOfRows * sizeof(double))); return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 7fb1ffbd64..ada5877d48 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -105,29 +105,14 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 3; - res->info.rows = rowNum; - res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + SSDataBlock *res = createDataBlock(); for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = TSDB_DATA_TYPE_NULL; - idata.info.bytes = 10; - idata.info.colId = i + 1; - - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); + SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, 1+i); + blockDataAppendColInfo(res, &idata); } - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 3; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3); + blockDataAppendColInfo(res, &idata); blockDataEnsureCapacity(res, rowNum); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); @@ -142,23 +127,18 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in rnode->slotId = 2; rnode->colId = 3; + res->info.rows = rowNum; *block = res; } else { SSDataBlock *res = *block; int32_t idx = taosArrayGetSize(res->pDataBlock); - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 1 + idx; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - res->info.numOfCols++; + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1+idx); + blockDataAppendColInfo(res, &idata); + blockDataEnsureCapacity(res, rowNum); + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - - colInfoDataEnsureCapacity(pColumn, 0, rowNum); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -399,7 +379,7 @@ TEST(columnTest, smallint_column_greater_double_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, true); - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -407,9 +387,9 @@ TEST(columnTest, smallint_column_greater_double_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -430,7 +410,7 @@ TEST(columnTest, int_column_greater_smallint_value) { flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); @@ -454,7 +434,7 @@ TEST(columnTest, int_column_greater_smallint_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, false); - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -462,9 +442,9 @@ TEST(columnTest, int_column_greater_smallint_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -478,11 +458,11 @@ TEST(columnTest, int_column_in_double_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; int32_t leftv[5] = {1, 2, 3, 4, 5}; double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; - bool eRes[5] = {true, true, true, false, false}; - SSDataBlock *src = NULL; + bool eRes[5] = {true, true, true, false, false}; + SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); nodesListAppend(list, pRight); @@ -498,7 +478,7 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -506,9 +486,9 @@ TEST(columnTest, int_column_in_double_list) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -521,8 +501,8 @@ TEST(columnTest, int_column_in_double_list) { TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; - bool eRes[5] = {true, true, false, false, false}; - SSDataBlock *src = NULL; + bool eRes[5] = {true, true, false, false, false}; + SSDataBlock *src = NULL; SScalarParam res = {0}; char leftv[5][5]= {0}; char rightv[3][5]= {0}; @@ -531,22 +511,22 @@ TEST(columnTest, binary_column_in_binary_list) { leftv[i][3] = 'b' + i; leftv[i][4] = '0' + i; varDataSetLen(leftv[i], 3); - } + } for (int32_t i = 0; i < 2; ++i) { rightv[i][2] = 'a' + i; rightv[i][3] = 'b' + i; rightv[i][4] = '0' + i; varDataSetLen(rightv[i], 3); - } + } for (int32_t i = 2; i < 3; ++i) { rightv[i][2] = 'a' + i; rightv[i][3] = 'a' + i; rightv[i][4] = 'a' + i; varDataSetLen(rightv[i], 3); } - + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); nodesListAppend(list, pRight); @@ -556,13 +536,13 @@ TEST(columnTest, binary_column_in_binary_list) { nodesListAppend(list, pRight); flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -570,9 +550,9 @@ TEST(columnTest, binary_column_in_binary_list) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -586,31 +566,31 @@ TEST(columnTest, binary_column_like_binary) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; char rightv[64] = {0}; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {true, false, true, false, true}; - + bool eRes[5] = {true, false, true, false, true}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = 'a'; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); sprintf(&rightv[2], "%s", "__0"); varDataSetLen(rightv, strlen(&rightv[2])); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -618,9 +598,9 @@ TEST(columnTest, binary_column_like_binary) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -633,31 +613,31 @@ TEST(columnTest, binary_column_like_binary) { TEST(columnTest, binary_column_is_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {false, false, true, false, true}; - + bool eRes[5] = {false, false, true, false, true}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); colDataAppend(pcolumn, 2, NULL, true); colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -665,9 +645,9 @@ TEST(columnTest, binary_column_is_null) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -680,31 +660,31 @@ TEST(columnTest, binary_column_is_null) { TEST(columnTest, binary_column_is_not_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {true, true, true, true, false}; - + bool eRes[5] = {true, true, true, true, false}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -712,9 +692,9 @@ TEST(columnTest, binary_column_is_not_null) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -735,13 +715,13 @@ TEST(opTest, smallint_column_greater_int_column) { flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -749,9 +729,9 @@ TEST(opTest, smallint_column_greater_int_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -773,13 +753,13 @@ TEST(opTest, smallint_value_add_int_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -787,9 +767,9 @@ TEST(opTest, smallint_value_add_int_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -816,13 +796,13 @@ TEST(opTest, bigint_column_multi_binary_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -830,9 +810,9 @@ TEST(opTest, bigint_column_multi_binary_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -859,13 +839,13 @@ TEST(opTest, smallint_column_and_binary_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -873,9 +853,9 @@ TEST(opTest, smallint_column_and_binary_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -897,13 +877,13 @@ TEST(opTest, smallint_column_or_float_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -911,9 +891,9 @@ TEST(opTest, smallint_column_or_float_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -935,13 +915,13 @@ TEST(opTest, smallint_column_or_double_value) { flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -949,9 +929,9 @@ TEST(opTest, smallint_column_or_double_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, true); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -964,28 +944,28 @@ TEST(opTest, smallint_column_or_double_value) { TEST(opTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {false, true, false, true, false}; - + bool eRes[5] = {false, true, false, true, false}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -993,9 +973,9 @@ TEST(opTest, binary_column_is_true) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1027,12 +1007,12 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1042,7 +1022,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); @@ -1050,13 +1030,13 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1064,9 +1044,9 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1096,13 +1076,13 @@ TEST(filterModelogicTest, same_column_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); - flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); @@ -1111,7 +1091,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); @@ -1119,13 +1099,13 @@ TEST(filterModelogicTest, same_column_and_or_and) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1133,9 +1113,9 @@ TEST(filterModelogicTest, same_column_and_or_and) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1165,12 +1145,12 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1180,7 +1160,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1188,13 +1168,13 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1202,9 +1182,9 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1234,12 +1214,12 @@ TEST(filterModelogicTest, same_column_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1249,7 +1229,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1257,13 +1237,13 @@ TEST(filterModelogicTest, same_column_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1271,9 +1251,9 @@ TEST(filterModelogicTest, same_column_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1305,12 +1285,12 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1320,7 +1300,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1328,13 +1308,13 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1342,7 +1322,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); for (int32_t i = 0; i < rowNum; ++i) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index d879150200..a6b7de71be 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -84,30 +84,25 @@ void scltInitLogFile() { void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { if (newBlock) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 1; - res->info.rows = rows; - res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + SSDataBlock *res = createDataBlock(); + SColumnInfoData idata = {0}; idata.info = *colInfo; - - taosArrayPush(res->pDataBlock, &idata); - taosArrayPush(pBlockList, &res); + blockDataAppendColInfo(res, &idata); blockDataEnsureCapacity(res, rows); + res->info.rows = rows; + + taosArrayPush(pBlockList, &res); *dataBlockId = taosArrayGetSize(pBlockList) - 1; res->info.blockId = *dataBlockId; *slotId = 0; } else { SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); - res->info.numOfCols++; SColumnInfoData idata = {0}; idata.info = *colInfo; - - colInfoDataEnsureCapacity(&idata, 0, rows); - - taosArrayPush(res->pDataBlock, &idata); + blockDataAppendColInfo(res, &idata); *dataBlockId = taosArrayGetSize(pBlockList) - 1; *slotId = taosArrayGetSize(res->pDataBlock) - 1; @@ -144,29 +139,18 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 3; + SSDataBlock *res = createDataBlock(); res->info.rows = rowNum; - res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = TSDB_DATA_TYPE_NULL; - idata.info.bytes = 10; - idata.info.colId = i + 1; - + SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, i + 1); int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); + blockDataAppendColInfo(res, &idata); } - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 3; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - colInfoDataEnsureCapacity(&idata, 0, rowNum); - taosArrayPush(res->pDataBlock, &idata); + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3); + blockDataAppendColInfo(res, &idata); + blockDataEnsureCapacity(res, rowNum); + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -185,17 +169,11 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SSDataBlock *res = *block; int32_t idx = taosArrayGetSize(res->pDataBlock); - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 1 + idx; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - res->info.numOfCols++; - SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1 + idx); + blockDataAppendColInfo(res, &idata); + blockDataEnsureCapacity(res, rowNum); - colInfoDataEnsureCapacity(pColumn, 0, rowNum); + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -2033,7 +2011,7 @@ void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t input->numOfRows = num; input->columnData->info = createColumnInfo(0, type, bytes); - colInfoDataEnsureCapacity(input->columnData, 0, num); + colInfoDataEnsureCapacity(input->columnData, num); if (setVal) { for (int32_t i = 0; i < num; ++i) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 1894f697c0..78179cb71e 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -74,10 +74,12 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; pRetrieve->numOfRows = htonl(pBlock->info.rows); - pRetrieve->numOfCols = htonl(pBlock->info.numOfCols); + + int32_t numOfCols = (int32_t) taosArrayGetSize(pBlock->pDataBlock); + pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); From a1401db4372f791a968d32d3a91e6477bee1b3b7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 17:23:38 +0800 Subject: [PATCH 04/54] fix(query): fix syntax error in unit test. --- source/libs/scalar/test/filter/filterTests.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index ada5877d48..8f9332cef8 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -921,7 +921,7 @@ TEST(opTest, smallint_column_or_double_value) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -965,7 +965,7 @@ TEST(opTest, binary_column_is_true) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1036,7 +1036,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1105,7 +1105,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1174,7 +1174,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1243,7 +1243,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1314,7 +1314,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { taosArrayGetSize(src->pDataBlock), src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); From aa4d86359844212f6f1051706cd73afdb21cbedd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 17:36:48 +0800 Subject: [PATCH 05/54] fix(query): fix syntax error. --- source/libs/command/src/command.c | 38 +++++++------------------ source/libs/executor/src/sortoperator.c | 2 +- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 91c3628db2..f8aa7ead8c 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -114,22 +114,12 @@ static int32_t execResetQueryCache() { return catalogClearCache(); } static SSDataBlock* buildCreateDBResultDataBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = SHOW_CREATE_DB_RESULT_COLS; - pBlock->info.hasVarCol = true; - - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD1_LEN; - - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD2_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); + SSDataBlock* pBlock = createDataBlock(); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1); + blockDataAppendColInfo(pBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); return pBlock; } @@ -271,21 +261,13 @@ static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveT } static SSDataBlock* buildCreateTbResultDataBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = SHOW_CREATE_TB_RESULT_COLS; - pBlock->info.hasVarCol = true; + SSDataBlock* pBlock = createDataBlock(); - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD1_LEN, 1); + blockDataAppendColInfo(pBlock, &infoData); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD1_LEN; - - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD2_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); return pBlock; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index a69f11f51d..3d8de1c1fd 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -286,7 +286,7 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; From c79eda9f707fc29dada5e8c6a515ddcfcbb5b698 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 19:34:56 +0800 Subject: [PATCH 06/54] fix(query): prepare the buffer before serialize the retrieved data. --- source/common/src/tdatablock.c | 65 +++++-------------------- source/libs/executor/src/executorimpl.c | 9 ++-- source/libs/executor/src/scanoperator.c | 3 +- 3 files changed, 16 insertions(+), 61 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7856f3a29b..b98001cb03 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -302,50 +302,20 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p ASSERT(pBlockInfo->capacity >= numOfRows); if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { -// if (pColumnInfoData->capacity < numOfRows) { -// char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); -// if (p == NULL) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// -// memset(p, 0, sizeof(int32_t)); -// pColumnInfoData->capacity = numOfRows; -// pColumnInfoData->varmeta.offset = (int32_t*)p; -// } - memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); -// if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { -// char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); -// if (tmp == NULL) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// -// pColumnInfoData->pData = tmp; -// pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; -// } + if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumnInfoData->pData = tmp; + pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; + } - memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); pColumnInfoData->varmeta.length = pSource->varmeta.length; + memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); } else { -// if (pColumnInfoData->capacity < numOfRows) { -// char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); -// if (tmp == NULL) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// -// memset(tmp, 0, BitmapLen(numOfRows)); -// pColumnInfoData->nullbitmap = tmp; -// -// int32_t newSize = numOfRows * pColumnInfoData->info.bytes; -// tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); -// if (tmp == NULL) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// -// pColumnInfoData->pData = tmp; -// pColumnInfoData->capacity = numOfRows; -// } - memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); } @@ -1874,8 +1844,6 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen } const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { - blockDataEnsureCapacity(pBlock, numOfRows); - const char* pStart = pData; int32_t dataLen = *(int32_t*)pStart; @@ -1884,11 +1852,6 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t pBlock->info.groupId = *(uint64_t*)pStart; pStart += sizeof(uint64_t); - if (pBlock->pDataBlock == NULL) { - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - taosArraySetSize(pBlock->pDataBlock, numOfCols); - } - for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData->info.type = *(int16_t*)pStart; @@ -1919,22 +1882,16 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); pStart += sizeof(int32_t) * numOfRows; - if (colLen[i] > 0) { + if (colLen[i] > 0 && pColInfoData->pData == NULL) { taosMemoryFreeClear(pColInfoData->pData); pColInfoData->pData = taosMemoryMalloc(colLen[i]); } } else { - if (pColInfoData->nullbitmap == NULL) { - pColInfoData->nullbitmap = taosMemoryCalloc(1, BitmapLen(numOfRows)); - } memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); pStart += BitmapLen(numOfRows); } if (colLen[i] > 0) { - if (pColInfoData->pData == NULL) { - pColInfoData->pData = taosMemoryCalloc(1, colLen[i]); - } memcpy(pColInfoData->pData, pStart, colLen[i]); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0c6e7fe5f6..9f917c9b81 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2075,14 +2075,13 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo blockCompressDecode(pBlock, numOfCols, numOfRows, pStart); blockDataEnsureCapacity(pRes, numOfRows); - relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - // data from mnode pRes->info.rows = numOfRows; + relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - taosArrayDestroy(pBlock->pDataBlock); - taosMemoryFree(pBlock); - // blockDataDestroy(pBlock); +// taosArrayDestroy(pBlock->pDataBlock); +// taosMemoryFree(pBlock); + blockDataDestroy(pBlock); } // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 49e821a71a..156be6ba13 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1366,7 +1366,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; const char* db = NULL; @@ -1558,7 +1557,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { if (pRsp->numOfRows == 0 || pRsp->completed) { pOperator->status = OP_EXEC_DONE; - qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64 " ", GET_TASKID(pTaskInfo), + qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), pRsp->numOfRows, pInfo->loadInfo.totalRows); if (pRsp->numOfRows == 0) { From b097e78708e41926175f2a8478168461724fe886 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 19:40:59 +0800 Subject: [PATCH 07/54] fix(query): clear buffer before set data. --- source/common/src/tdatablock.c | 2 +- source/libs/executor/src/executorimpl.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b98001cb03..4a9d573881 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1201,7 +1201,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { pBlock->info.rows = 0; pBlock->info.capacity = 0; - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData colInfo = {.hasNull = true, .info = p->info}; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9f917c9b81..db107d444b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2047,6 +2047,8 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList) { if (pColList == NULL) { // data from other sources + blockDataCleanup(pRes); + blockDataEnsureCapacity(pRes, numOfRows); blockCompressDecode(pRes, numOfOutput, numOfRows, pData); pRes->info.rows = numOfRows; } else { // extract data according to pColList From 0a884d58bbc052482ec3c4ea73935772b515607d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 19:41:57 +0800 Subject: [PATCH 08/54] refactor: remove unnecessary code. --- source/libs/executor/src/executorimpl.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index db107d444b..bd99fd0be3 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2050,7 +2050,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo blockDataCleanup(pRes); blockDataEnsureCapacity(pRes, numOfRows); blockCompressDecode(pRes, numOfOutput, numOfRows, pData); - pRes->info.rows = numOfRows; } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; @@ -2080,9 +2079,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo // data from mnode pRes->info.rows = numOfRows; relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - -// taosArrayDestroy(pBlock->pDataBlock); -// taosMemoryFree(pBlock); blockDataDestroy(pBlock); } From 72a942b09ec311556748b06518ab2a91df7f160b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 19:44:14 +0800 Subject: [PATCH 09/54] refactor(query): do internal refactor. --- include/common/tdatablock.h | 2 +- source/common/src/tdatablock.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index d65fdc0675..7e9dd43bae 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -183,7 +183,7 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u int32_t getJsonValueLen(const char* data); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4a9d573881..5d6dde04ca 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -214,7 +214,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } } -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRow2 == 0) { From 324eecdd998d9527d83f98f32678bc3a8abc7047 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 20:24:26 +0800 Subject: [PATCH 10/54] fix(query): prepare the buffer before assigning result. --- source/common/src/tdatablock.c | 4 +++- source/libs/scalar/src/scalar.c | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5d6dde04ca..57d75770cf 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -299,7 +299,9 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p return numOfRows; } - ASSERT(pBlockInfo->capacity >= numOfRows); + if (pBlockInfo != NULL) { + ASSERT(pBlockInfo->capacity >= numOfRows); + } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 9890cff6f2..75a9bc3809 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -982,7 +982,8 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + + colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows); colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); pDst->numOfRows = res->numOfRows; taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); From ed8002f903f469636ce69822e1a7fccd98045db0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 20:34:24 +0800 Subject: [PATCH 11/54] fix(query): fix syntax error. --- source/libs/stream/src/streamDispatch.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index d26100d455..b5a9e221a7 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -96,16 +96,17 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve = taosMemoryCalloc(1, dataStrLen); if (pRetrieve == NULL) return -1; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); pRetrieve->useconds = 0; pRetrieve->precision = TSDB_DEFAULT_PRECISION; pRetrieve->compressed = 0; pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; pRetrieve->numOfRows = htonl(pBlock->info.rows); - pRetrieve->numOfCols = htonl(pBlock->info.numOfCols); + pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); SStreamRetrieveReq req = { .streamId = pTask->streamId, From 02a53ee81f8fb635035f51682e6a80f47ab9686d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 21:03:59 +0800 Subject: [PATCH 12/54] fix(query): set correct memory buffer size. --- source/common/src/tdatablock.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 57d75770cf..4f513335eb 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1169,7 +1169,9 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { return TSDB_CODE_SUCCESS; } - pDataBlock->info.capacity = numOfRows; + if (pDataBlock->info.capacity < numOfRows) { + pDataBlock->info.capacity = numOfRows; + } size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { @@ -1878,15 +1880,21 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - pColInfoData->varmeta.length = colLen[i]; - pColInfoData->varmeta.allocLen = colLen[i]; +// pColInfoData->varmeta.length = colLen[i]; +// pColInfoData->varmeta.allocLen = colLen[i]; memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); pStart += sizeof(int32_t) * numOfRows; - if (colLen[i] > 0 && pColInfoData->pData == NULL) { - taosMemoryFreeClear(pColInfoData->pData); - pColInfoData->pData = taosMemoryMalloc(colLen[i]); + if (colLen[i] > 0 && pColInfoData->varmeta.allocLen < colLen[i]) { + char* tmp = taosMemoryRealloc(pColInfoData->pData, colLen[i]); + if (tmp == NULL) { + return NULL; + } + + pColInfoData->pData = tmp; + pColInfoData->varmeta.allocLen = colLen[i]; + pColInfoData->varmeta.length = colLen[i]; } } else { memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); From ecc05bf75c540a1cccd34682659ab1858da46325 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 21:12:08 +0800 Subject: [PATCH 13/54] fix(query): check pointer before use it. --- source/common/src/tdatablock.c | 3 --- source/libs/function/src/builtinsimpl.c | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4f513335eb..168d949631 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1880,9 +1880,6 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { -// pColInfoData->varmeta.length = colLen[i]; -// pColInfoData->varmeta.allocLen = colLen[i]; - memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); pStart += sizeof(int32_t) * numOfRows; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4d869705ad..b0de3b3364 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1450,8 +1450,8 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple int32_t pageId = pTuplePos->pageId; int32_t offset = pTuplePos->offset; - int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock); if (pTuplePos->pageId != -1) { + int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock); SFilePage* pPage = getBufPage(pCtx->pBuf, pageId); bool* nullList = (bool*)((char*)pPage + offset); From 300f35188e0c2887bb6d25f5805e131a29f80733 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 21:18:11 +0800 Subject: [PATCH 14/54] fix(query): prepare the SColumnInfoData. --- source/common/src/tdatablock.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 168d949631..93d3661caf 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1856,6 +1856,11 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t pBlock->info.groupId = *(uint64_t*)pStart; pStart += sizeof(uint64_t); + if (pBlock->pDataBlock == NULL) { + pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + taosArraySetSize(pBlock->pDataBlock, numOfCols); + } + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData->info.type = *(int16_t*)pStart; From d3d258caa9d791d0e103b13026a029a7cc5938c1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 23:05:34 +0800 Subject: [PATCH 15/54] fix(query): prepare buffer before copying data. --- source/libs/executor/src/sortoperator.c | 8 +++++--- source/libs/function/src/tudf.c | 1 - tests/script/general/parser/udf.sim | 1 + tests/script/tsim/query/udf.sim | 1 + 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 3d8de1c1fd..79d5fe104f 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -120,6 +120,8 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i } if (p->info.rows > 0) { + blockDataEnsureCapacity(pDataBlock, capacity); + // todo extract function to handle this int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { @@ -132,7 +134,6 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i } pDataBlock->info.rows = p->info.rows; - pDataBlock->info.capacity = p->info.rows; } blockDataDestroy(p); @@ -258,6 +259,7 @@ typedef struct SGroupSortOperatorInfo { SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SGroupSortOperatorInfo* pInfo) { blockDataCleanup(pDataBlock); + blockDataEnsureCapacity(pDataBlock, capacity); SSDataBlock* p = tsortGetSortedDataBlock(pHandle); if (p == NULL) { @@ -571,7 +573,8 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData } } - if (p->info.rows > 0) { + if (p->info.rows > 0) {// todo extract method + blockDataEnsureCapacity(pDataBlock, p->info.rows); int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); @@ -583,7 +586,6 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData } pDataBlock->info.rows = p->info.rows; - pDataBlock->info.capacity = p->info.rows; pDataBlock->info.groupId = pInfo->groupId; } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index ea2a7567e4..aead5f2867 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1081,7 +1081,6 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; - SSDataBlock* pTempBlock = createDataBlock(); pTempBlock->info.rows = pInput->totalRows; pTempBlock->info.uid = pInput->uid; diff --git a/tests/script/general/parser/udf.sim b/tests/script/general/parser/udf.sim index c048a7c37a..4b767dbf60 100644 --- a/tests/script/general/parser/udf.sim +++ b/tests/script/general/parser/udf.sim @@ -44,6 +44,7 @@ endi sql create function n02 as '/tmp/normal' outputtype bool; sql show functions; if $rows != 1 then + print expect 1, actual $rows return -1 endi diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index d9821a0495..4e95095172 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -44,6 +44,7 @@ endi sql select udf2(f) from t; if $rows != 1 then + print expect 1, actual $rows return -1 endi if $data00 != 2.236067977 then From 32e479858fe6de8ac0671aa2f24c9491367f186b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Jun 2022 23:48:23 +0800 Subject: [PATCH 16/54] fix(query): disable type check. --- source/common/src/tdatablock.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 93d3661caf..0fab1c4f5a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1268,7 +1268,8 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoDat return terrno; } - ASSERT(pColInfoData->info.type != 0); + // todo disable it temporarily +// ASSERT(pColInfoData->info.type != 0); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { pBlock->info.hasVarCol = true; } From a0795fdd10b7794f4bd9dede1c97f0bfab950b46 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 00:47:28 +0800 Subject: [PATCH 17/54] fix(query): set correct varchar length. --- source/common/src/tdatablock.c | 3 ++- source/libs/executor/src/executorimpl.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 0fab1c4f5a..565f8da0a0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1897,8 +1897,9 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t pColInfoData->pData = tmp; pColInfoData->varmeta.allocLen = colLen[i]; - pColInfoData->varmeta.length = colLen[i]; } + + pColInfoData->varmeta.length = colLen[i]; } else { memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); pStart += BitmapLen(numOfRows); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ed867bed45..188edaa68f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2048,7 +2048,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo SArray* pColList) { if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); - blockDataEnsureCapacity(pRes, numOfRows); +// blockDataEnsureCapacity(pRes, numOfRows); blockCompressDecode(pRes, numOfOutput, numOfRows, pData); } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); From 673261a1bd1dfbdd9d7c85df41371c883f5f606b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 09:42:29 +0800 Subject: [PATCH 18/54] fix(query): set correct varchar length. --- source/libs/executor/src/scanoperator.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b978d54a5d..e1b9e7e3de 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2005,6 +2005,7 @@ _error: return code; } +// todo refactor or remove it static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2145,7 +2146,13 @@ static SSDataBlock* getTableDataBlock(void* param) { continue; } - tsdbRetrieveDataBlockInfo(reader, &pBlock->info); + blockDataCleanup(pBlock); + SDataBlockInfo binfo = pBlock->info; + tsdbRetrieveDataBlockInfo(reader, &binfo); + + binfo.capacity = binfo.rows; + blockDataEnsureCapacity(pBlock, binfo.capacity); + pBlock->info = binfo; uint32_t status = 0; int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status); From 0457fab0779f9ab5bc3c393cb1a1b191f6e270fb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 10:03:43 +0800 Subject: [PATCH 19/54] fix(query): adjust the capacity of ssdatablock. --- include/common/tcommon.h | 2 +- source/libs/executor/src/scanoperator.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 9c0c3c0616..f841a7d4cd 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -68,7 +68,7 @@ typedef struct SDataBlockInfo { STimeWindow window; int32_t rows; int32_t rowSize; - int64_t uid; // the uid of table, from which current data block comes + uint64_t uid; // the uid of table, from which current data block comes uint16_t blockId; // block id, generated by physical planner uint64_t groupId; // no need to serialize int16_t hasVarCol; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e1b9e7e3de..b1bd50daf0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -969,6 +969,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { uint64_t uid = 0; int32_t numOfRows = 0; + // todo refactor int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows); if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { @@ -976,12 +977,11 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { return NULL; } - blockDataEnsureCapacity(pInfo->pRes, numOfRows); - pInfo->pRes->info.groupId = groupId; pInfo->pRes->info.rows = numOfRows; pInfo->pRes->info.uid = uid; pInfo->pRes->info.type = STREAM_NORMAL; + pInfo->pRes->info.capacity = numOfRows; // for generating rollup SMA result, each time is an independent time serie. // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this From ce7e7fe23d64fb75ab1d885e759a71c0e07be02c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 10:13:35 +0800 Subject: [PATCH 20/54] refactor:remove some debug info. --- source/libs/executor/src/executor.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e805de3727..cf72dfd796 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -68,8 +68,6 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayClear(p->pDataBlock); taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock); - printf("size------------->%ld, %ld, %p, \n", taosArrayGetSize(pDataBlock->pDataBlock), taosArrayGetSize(pDataBlock->pDataBlock) * pDataBlock->pDataBlock->elemSize, - pDataBlock->pDataBlock); taosArrayPush(pInfo->pBlockLists, &p); } } else { From 4576b0a94d7d2a30224d893b5fbb2fd2d18bb28e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 11:07:14 +0800 Subject: [PATCH 21/54] fix(query): fix syntax error. --- source/libs/executor/inc/executorimpl.h | 4 ++-- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/scanoperator.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 9bcfae549d..ccdbe5c890 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -838,8 +838,8 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, - SNode* pTagCond); + STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); + SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ad5ebb5974..cd3b8cb705 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4009,7 +4009,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; - createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId, pTagCond); + createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); STableScanInfo* pScanInfo = pOperator->info; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d1d0a70115..2e3ae851bf 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1956,7 +1956,7 @@ int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { } int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, SNode* pTagCond) { + STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) { int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { return code; From 91eb4ff88b51f4ed88ae5031df52512676f54efb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 11:32:21 +0800 Subject: [PATCH 22/54] test(query): update test case. --- tests/system-test/2-query/json_tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index ff7adfc759..6cd3f49636 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -509,7 +509,7 @@ class TDTestCase: tdSql.checkData(0, 0, 5.3) # tdSql.query("select twa(dataint) from jsons1 where jtag is not null") # tdSql.checkData(0, 0, 36) - tdSql.error("select irate(dataint) from jsons1 where jtag is not null") + tdSql.query("select irate(dataint) from jsons1 where jtag is not null") tdSql.query("select sum(dataint) from jsons1 where jtag->'tag1' is not null") tdSql.checkData(0, 0, 45) tdSql.query("select stddev(dataint) from jsons1 where jtag->'tag1'>1") From 1a1704f26ba8c71e0ba115b92a845c0057ffd2ad Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 12:59:33 +0800 Subject: [PATCH 23/54] refactor(query): refactor taosParseTime to support ISO8601 date-time format and timestamp shortcut format TD-16770 --- source/common/src/ttime.c | 58 +++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 0b59e9b6cc..8fbdeb0654 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -76,22 +76,28 @@ void deltaToUtcInitOnce() { static int64_t parseFraction(char* str, char** end, int32_t timePrec); static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); -static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec); -static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec); +static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); +static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static char* forwardToTimeStringEnd(char* str); static bool checkTzPresent(const char* str, int32_t len); -static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec) = {parseLocaltime, - parseLocaltimeDst}; +static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) = {parseLocaltime, + parseLocaltimeDst}; int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t timePrec, int8_t day_light) { /* parse datatime string in with tz */ if (strnchr(timestr, 'T', len, false) != NULL) { - return parseTimeWithTz(timestr, utime, timePrec, 'T'); - } else if (checkTzPresent(timestr, len)) { - return parseTimeWithTz(timestr, utime, timePrec, 0); + if (checkTzPresent(timestr, len)) { + return parseTimeWithTz(timestr, utime, timePrec, 'T'); + } else { + return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 'T'); + } } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec); + if (checkTzPresent(timestr, len)) { + return parseTimeWithTz(timestr, utime, timePrec, 0); + } else { + return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 0); + } } } @@ -333,13 +339,25 @@ static FORCE_INLINE bool validateTm(struct tm* pTm) { return true; } -int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePrec) { +int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePrec, char delim) { *time = 0; struct tm tm = {0}; - char* str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm); + char *str; + if (delim == 'T') { + str = taosStrpTime(timestr, "%Y-%m-%dT%H:%M:%S", &tm); + } else if (delim == 0) { + str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm); + } else { + str = NULL; + } + if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { - return -1; + //if parse failed, try "%Y-%m-%d" format + str = taosStrpTime(timestr, "%Y-%m-%d", &tm); + if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { + return -1; + } } #ifdef _MSC_VER @@ -367,14 +385,26 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePr return 0; } -int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* time, int32_t timePrec) { +int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* time, int32_t timePrec, char delim) { *time = 0; struct tm tm = {0}; tm.tm_isdst = -1; - char* str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm); + char *str; + if (delim == 'T') { + str = taosStrpTime(timestr, "%Y-%m-%dT%H:%M:%S", &tm); + } else if (delim == 0) { + str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm); + } else { + str = NULL; + } + if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { - return -1; + //if parse failed, try "%Y-%m-%d" format + str = taosStrpTime(timestr, "%Y-%m-%d", &tm); + if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { + return -1; + } } /* mktime will be affected by TZ, set by using taos_options */ From 34daf19946bdceeaf4488a8b7cb252b12bea5374 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 11:11:55 +0800 Subject: [PATCH 24/54] feat: let drop dnode support ep --- source/dnode/mnode/impl/src/mndDnode.c | 35 +++++++++++++------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index e58fe0835a..8c50366dc1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -592,30 +592,30 @@ _OVER: } static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SDnodeObj *pDnode = NULL; - SMnodeObj *pMObj = NULL; - SQnodeObj *pQObj = NULL; - SSnodeObj *pSObj = NULL; - SMDropMnodeReq dropReq = {0}; + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SDnodeObj *pDnode = NULL; + SMnodeObj *pMObj = NULL; + SQnodeObj *pQObj = NULL; + SSnodeObj *pSObj = NULL; + SDropDnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + if (tDeserializeSDropDnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } - mInfo("dnode:%d, start to drop", dropReq.dnodeId); - - if (dropReq.dnodeId <= 0) { - terrno = TSDB_CODE_MND_INVALID_DNODE_ID; - goto _OVER; - } + mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port); pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId); if (pDnode == NULL) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - goto _OVER; + char ep[TSDB_EP_LEN + 1] = {0}; + snprintf(ep, sizeof(ep), dropReq.fqdn, dropReq.port); + pDnode = mndAcquireDnodeByEp(pMnode, ep); + if (pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + goto _OVER; + } } pQObj = mndAcquireQnode(pMnode, dropReq.dnodeId); @@ -670,6 +670,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { return -1; } + mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); SDnodeObj *pDnode = mndAcquireDnode(pMnode, cfgReq.dnodeId); if (pDnode == NULL) { mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr()); @@ -686,7 +687,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { tSerializeSMCfgDnodeReq(pBuf, bufLen, &cfgReq); mDebug("dnode:%d, send config req to dnode, app:%p", cfgReq.dnodeId, pReq->info.ahandle); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .info = pReq->info}; + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; return tmsgSendReq(&epSet, &rpcMsg); } From 4ed47a029642bd01be87e4a994f39673e962fecc Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 23 Jun 2022 13:35:24 +0800 Subject: [PATCH 25/54] feat(tmq): scan tsdb as snapshot --- examples/c/tmq.c | 33 ++++++++++++++++--- include/common/tcommon.h | 4 +-- include/common/tmsg.h | 11 ++++--- include/libs/executor/executor.h | 9 +++++ source/client/src/tmq.c | 17 ++++++++++ source/dnode/mnode/impl/src/mndScheduler.c | 2 +- source/dnode/vnode/src/inc/tq.h | 1 + source/dnode/vnode/src/tq/tq.c | 38 ++++++++++++++++++---- source/dnode/vnode/src/tq/tqExec.c | 24 ++++++++++++++ source/libs/executor/src/executor.c | 19 ++++++----- source/libs/executor/src/executorimpl.c | 16 ++++----- source/libs/executor/src/scanoperator.c | 11 ++++++- source/libs/stream/src/streamExec.c | 1 - 13 files changed, 148 insertions(+), 38 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 7870d7d9a1..980c86b877 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -76,19 +76,41 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create table if not exists ct1 using st1 tags(2000)"); + pRes = taos_query(pConn, "insert into ct0 values(now, 1, 2, 'a')"); if (taos_errno(pRes) != 0) { - printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); return -1; } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct1 using st1 tags(2000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct1 values(now, 3, 4, 'b')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); pRes = taos_query(pConn, "create table if not exists ct3 using st1 tags(3000)"); if (taos_errno(pRes) != 0) { - printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes)); + printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); return -1; } - taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 values(now, 5, 6, 'c')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + return 0; } @@ -168,6 +190,9 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); + + tmq_conf_set(conf, "experiment.use.snapshot", "true"); + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 1b32dc26f5..c36dc51220 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -113,8 +113,8 @@ typedef struct SQueryTableDataCond { int32_t type; // data block load type: int32_t numOfTWindows; STimeWindow* twindows; - int32_t startVersion; - int32_t endVersion; + int64_t startVersion; + int64_t endVersion; } SQueryTableDataCond; void* blockDataDestroy(SSDataBlock* pBlock); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f654d523f3..c9307ab912 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -696,12 +696,12 @@ typedef struct { typedef STableCfg STableCfgRsp; -int32_t tSerializeSTableCfgReq(void *buf, int32_t bufLen, STableCfgReq *pReq); -int32_t tDeserializeSTableCfgReq(void *buf, int32_t bufLen, STableCfgReq *pReq); +int32_t tSerializeSTableCfgReq(void* buf, int32_t bufLen, STableCfgReq* pReq); +int32_t tDeserializeSTableCfgReq(void* buf, int32_t bufLen, STableCfgReq* pReq); -int32_t tSerializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp); -int32_t tDeserializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp); -void tFreeSTableCfgRsp(STableCfgRsp *pRsp); +int32_t tSerializeSTableCfgRsp(void* buf, int32_t bufLen, STableCfgRsp* pRsp); +int32_t tDeserializeSTableCfgRsp(void* buf, int32_t bufLen, STableCfgRsp* pRsp); +void tFreeSTableCfgRsp(STableCfgRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; @@ -2652,6 +2652,7 @@ typedef struct { SMsgHead head; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; int8_t withTbName; + int8_t useSnapshot; int32_t epoch; uint64_t reqId; int64_t consumerId; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 083f6ae1b0..6738fc23bc 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -36,11 +36,13 @@ typedef struct SReadHandle { void* vnode; void* mnd; SMsgCb* pMsgCb; + int8_t initTsdbReader; } SReadHandle; enum { STREAM_DATA_TYPE_SUBMIT_BLOCK = 1, STREAM_DATA_TYPE_SSDATA_BLOCK = 2, + STREAM_DATA_TYPE_FROM_SNAPSHOT = 3, }; typedef enum { @@ -56,6 +58,13 @@ typedef enum { */ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle); +/** + * Switch the stream scan to snapshot mode + * @param tinfo + * @return + */ +int32_t qStreamScanSnapshot(qTaskInfo_t tinfo); + /** * Set the input data block for the stream scan. * @param tinfo diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 3c349f61a1..949965296f 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -54,6 +54,7 @@ struct tmq_conf_t { int8_t autoCommit; int8_t resetOffset; int8_t withTbName; + int8_t useSnapshot; uint16_t port; int32_t autoCommitInterval; char* ip; @@ -69,6 +70,7 @@ struct tmq_t { char groupId[TSDB_CGROUP_LEN]; char clientId[256]; int8_t withTbName; + int8_t useSnapshot; int8_t autoCommit; int32_t autoCommitInterval; int32_t resetOffsetCfg; @@ -282,6 +284,18 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } + if (strcmp(key, "experiment.use.snapshot") == 0) { + if (strcmp(value, "true") == 0) { + conf->useSnapshot = true; + return TMQ_CONF_OK; + } else if (strcmp(value, "false") == 0) { + conf->useSnapshot = false; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + } + if (strcmp(key, "td.connect.ip") == 0) { conf->ip = strdup(value); return TMQ_CONF_OK; @@ -953,6 +967,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); pTmq->withTbName = conf->withTbName; + pTmq->useSnapshot = conf->useSnapshot; pTmq->autoCommit = conf->autoCommit; pTmq->autoCommitInterval = conf->autoCommitInterval; pTmq->commitCb = conf->commitCb; @@ -1534,6 +1549,8 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pReq->currentOffset = reqOffset; pReq->reqId = generateRequestId(); + pReq->useSnapshot = tmq->useSnapshot; + pReq->head.vgId = htonl(pVg->vgId); pReq->head.contLen = htonl(sizeof(SMqPollReq)); return pReq; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index b1729c4047..9d8ed3701e 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -509,7 +509,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SStreamTask* pTask = tNewSStreamTask(pStream->uid); mndAddTaskToTaskSet(taskOneLevel, pTask); - // input + // source pTask->isDataScan = 1; // trigger diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 8f6077e996..59c3e95b9c 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -150,6 +150,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead* // tqExec int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId); +int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId); int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp); // tqMeta diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 046ee64878..c531e9ee91 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -227,19 +227,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch); } - SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048); - if (pHeadWithCkSum == NULL) { - return -1; - } - - walSetReaderCapacity(pHandle->pWalReader, 2048); - SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pReq->currentOffset; rsp.blockData = taosArrayInit(0, sizeof(void*)); rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); + if (rsp.blockData == NULL || rsp.blockDataLen == NULL) { + return -1; + } + rsp.withTbName = pReq->withTbName; if (rsp.withTbName) { rsp.blockTbName = taosArrayInit(0, sizeof(void*)); @@ -253,6 +250,28 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { rsp.blockSchema = taosArrayInit(0, sizeof(void*)); } +#if 1 + if (pReq->useSnapshot) { + int64_t lastVer = walGetCommittedVer(pTq->pWal); + if (rsp.reqOffset < lastVer) { + tqScanSnapshot(pTq, &pHandle->execHandle, &rsp, workerId); + + if (rsp.blockNum != 0) { + rsp.withTbName = false; + rsp.rspOffset = lastVer; + goto SEND_RSP; + } + } + } +#endif + + SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048); + if (pHeadWithCkSum == NULL) { + return -1; + } + + walSetReaderCapacity(pHandle->pWalReader, 2048); + while (1) { consumerEpoch = atomic_load_32(&pHandle->epoch); if (consumerEpoch > reqEpoch) { @@ -292,6 +311,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { metaRsp.metaRsp = pHead->body; if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { code = -1; + goto OVER; } code = 0; goto OVER; @@ -308,6 +328,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { taosMemoryFree(pHeadWithCkSum); +SEND_RSP: ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum); ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum); if (rsp.withSchema) { @@ -376,6 +397,8 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { SReadHandle handle = { .reader = pHandle->execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, + .vnode = pTq->pVnode, + .initTsdbReader = 1, }; pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); ASSERT(pHandle->execHandle.execCol.task[i]); @@ -448,6 +471,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { .reader = pStreamReader, .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, + .initTsdbReader = 1, }; /*pTask->exec.inputHandle = pStreamReader;*/ pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 7c75d88a83..e6f960c8c1 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -60,6 +60,30 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqD return 0; } +int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId) { + ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN); + qTaskInfo_t task = pExec->execCol.task[workerId]; + if (qStreamScanSnapshot(task) < 0) { + ASSERT(0); + } + while (1) { + SSDataBlock* pDataBlock = NULL; + uint64_t ts = 0; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + ASSERT(0); + } + if (pDataBlock == NULL) break; + + ASSERT(pDataBlock->info.rows != 0); + ASSERT(pDataBlock->info.numOfCols != 0); + + tqAddBlockDataToRsp(pDataBlock, pRsp); + pRsp->blockNum++; + } + + return 0; +} + int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId) { if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) { qTaskInfo_t task = pExec->execCol.task[workerId]; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6b93119987..1bd1ce14b1 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -67,6 +67,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock); taosArrayPush(pInfo->pBlockLists, &p); } + } else if (type == STREAM_DATA_TYPE_FROM_SNAPSHOT) { + // do nothing + ASSERT(pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT); } else { ASSERT(0); } @@ -75,6 +78,14 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } +int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) { + if (tinfo == NULL) { + return TSDB_CODE_QRY_APP_ERROR; + } + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_DATA_TYPE_FROM_SNAPSHOT, 0, NULL); +} + int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) { return qSetMultiStreamInput(tinfo, input, 1, type, assignUid); } @@ -106,14 +117,6 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { return NULL; } - // print those info into log -#if 0 - pMsg->sId = pMsg->sId; - pMsg->queryId = pMsg->queryId; - pMsg->taskId = pMsg->taskId; - pMsg->contentLen = pMsg->contentLen; -#endif - /*qDebugL("stream task string %s", (const char*)msg);*/ struct SSubplan* plan = NULL; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 828da0bcf8..39ce19ed90 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1340,7 +1340,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR } if (rowRes != NULL) { - int32_t totalRows = pBlock->info.rows; + int32_t totalRows = pBlock->info.rows; SSDataBlock* px = createOneDataBlock(pBlock, true); for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { @@ -3968,7 +3968,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } } } - int32_t len = (int32_t)(pStart - (char*)keyBuf); + int32_t len = (int32_t)(pStart - (char*)keyBuf); uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len); @@ -4047,12 +4047,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo tsdbReaderT pDataReader = NULL; if (pHandle) { - if (pHandle->vnode) { - // for stram + if (pHandle->initTsdbReader) { + // for stream + ASSERT(pHandle->vnode); pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); } else { // for tq + ASSERT(pHandle->meta); getTableList(pHandle->meta, pScanPhyNode, pTableListInfo, pTagCond); } } @@ -4502,16 +4504,12 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead (*pTaskInfo)->sql = sql; (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond); + if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; goto _complete; } - if ((*pTaskInfo)->pRoot == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _complete; - } - return code; _complete: diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 45f80c3c6f..1d1e0d6bb8 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -928,7 +928,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); blockDataUpdateTsWindow(pBlock, 0); return pBlock; - } else { + } else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { blockDataDestroy(pInfo->pUpdateRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; @@ -1062,6 +1062,15 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; + } else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) { + SSDataBlock* pResult = doTableScan(pInfo->pOperatorDumy); + if (pResult) { + return pResult->info.rows > 0 ? pResult : NULL; + } + return NULL; + } else { + ASSERT(0); + return NULL; } } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 69cb5b43fa..6fada59c84 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -80,7 +80,6 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { } qRes->type = STREAM_INPUT__DATA_BLOCK; qRes->blocks = pRes; - /*qRes->sourceVg = pTask->nodeId;*/ if (streamTaskOutput(pTask, qRes) < 0) { streamQueueProcessFail(pTask->inputQueue); taosArrayDestroy(pRes); From a31569f116b8bc22266a9ff7eb3e4b73c70468a3 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 13:38:20 +0800 Subject: [PATCH 26/54] fix: set correct datablock rows and columns --- source/common/src/tdatablock.c | 2 +- source/libs/function/src/tudf.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 565f8da0a0..40e21fc6ac 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -463,7 +463,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 } pDst->info = pBlock->info; - + pDst->info.rows = 0; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index aead5f2867..da9474ede0 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -763,6 +763,7 @@ void freeUdfInterBuf(SUdfInterBuf *buf) { int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) { udfBlock->numOfRows = block->info.rows; + udfBlock->numOfCols = taosArrayGetSize(block->pDataBlock); udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn*)); for (int32_t i = 0; i < udfBlock->numOfCols; ++i) { udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn)); From 044f27bbca0746007f20ab900b7711fef1493254 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 13:58:36 +0800 Subject: [PATCH 27/54] fix(query): avg/stddev divide 0 handling TD-16762 --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index d7ff03ec59..647f039d96 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -927,7 +927,7 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { // check for overflow if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) { - GET_RES_INFO(pCtx)->isNullRes = 1; + GET_RES_INFO(pCtx)->numOfRes = 0; } return functionFinalize(pCtx, pBlock); From edc42a78abc85235f8f53e9025062db19c1e9a9a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 13:59:31 +0800 Subject: [PATCH 28/54] fix: fix compilation error --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 12 ++++++------ source/libs/command/src/command.c | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 9c8918a445..b9bd49516a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -203,7 +203,6 @@ SSDataBlock* dmBuildVariablesBlock(void) { taosArrayPush(pBlock->pDataBlock, &colInfoData); } - pBlock->info.numOfCols = pMeta[index].colNum; pBlock->info.hasVarCol = true; return pBlock; @@ -262,8 +261,9 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId); - size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pBlock->info.numOfCols + - blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols + + blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(numOfCols); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { @@ -274,10 +274,10 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } char *pStart = pRsp->data; - *(int32_t *)pStart = htonl(pBlock->info.numOfCols); + *(int32_t *)pStart = htonl(numOfCols); pStart += sizeof(int32_t); // number of columns - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SSysTableSchema *pSchema = (SSysTableSchema *)pStart; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); @@ -289,7 +289,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t len = 0; - blockCompressEncode(pBlock, pStart, &len, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pStart, &len, numOfCols, false); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 2ba0f1a77d..8d01ed7315 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -524,10 +524,9 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return TSDB_CODE_FAILED; static SSDataBlock* buildLocalVariablesResultDataBlock() { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = SHOW_LOCAL_VARIABLES_RESULT_COLS; pBlock->info.hasVarCol = true; - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); + pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); SColumnInfoData infoData = {0}; infoData.info.type = TSDB_DATA_TYPE_VARCHAR; From cfad058333c69398337643f259f099a5c4da79b0 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 13:58:36 +0800 Subject: [PATCH 29/54] fix(query): avg/stddev divide 0 handling TD-16762 --- source/libs/function/src/builtinsimpl.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 647f039d96..5f3d1b89db 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1774,6 +1774,11 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pStddevRes->result = sqrt(fabs(pStddevRes->quadraticDSum / ((double)pStddevRes->count) - avg * avg)); } + // check for overflow + if (isinf(pStddevRes->result) || isnan(pStddevRes->result)) { + GET_RES_INFO(pCtx)->numOfRes = 0; + } + return functionFinalize(pCtx, pBlock); } From c4b1d43bbc9f01ecdcd3045053a57a03e97f7ed6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 14:12:03 +0800 Subject: [PATCH 30/54] fix: show dnode 1 variables count --- tests/script/tsim/show/basic.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index 8f9362fc69..e2138e0008 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -233,7 +233,7 @@ if $rows != 4 then endi sql show dnode 1 variables; -if $rows != 114 then +if $rows != 116 then return -1 endi From d5a27d50e321406020d6cd88d8a5517fe0ad1e53 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 13:58:36 +0800 Subject: [PATCH 31/54] fix(query): avg/stddev divide 0 handling TD-16762 --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5f3d1b89db..690319a4cf 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -561,7 +561,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { // check for overflow if (IS_FLOAT_TYPE(type) && (isinf(pSumRes->dsum) || isnan(pSumRes->dsum))) { - GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElems = 0; } _sum_over: From 0a24fbcdd6a0af4c5ceec2a2d830bc062a845297 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 23 Jun 2022 14:19:21 +0800 Subject: [PATCH 32/54] feat: persistence of rsma qtaskinfo --- include/util/taoserror.h | 1 + source/dnode/vnode/CMakeLists.txt | 1 + source/dnode/vnode/src/inc/sma.h | 119 ++++++++--- source/dnode/vnode/src/meta/metaQuery.c | 4 +- source/dnode/vnode/src/sma/smaEnv.c | 51 +++-- source/dnode/vnode/src/sma/smaOpen.c | 13 ++ source/dnode/vnode/src/sma/smaRollup.c | 166 ++++++++++++--- source/dnode/vnode/src/sma/smaTimeRange.c | 4 +- source/dnode/vnode/src/sma/smaUtil.c | 236 ++++++++++++++++++++++ source/util/src/terror.c | 3 +- 10 files changed, 514 insertions(+), 84 deletions(-) create mode 100644 source/dnode/vnode/src/sma/smaUtil.c diff --git a/include/util/taoserror.h b/include/util/taoserror.h index eb68f52a40..8af5945300 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -71,6 +71,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE TAOS_DEF_ERROR_CODE(0, 0x0029) #define TSDB_CODE_INVALID_TIMESTAMP TAOS_DEF_ERROR_CODE(0, 0x0030) #define TSDB_CODE_MSG_DECODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0031) +#define TSDB_CODE_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0032) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0040) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0041) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 15eb35c700..b09ab34224 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -29,6 +29,7 @@ target_sources( # sma "src/sma/sma.c" "src/sma/smaEnv.c" + "src/sma/smaUtil.c" "src/sma/smaOpen.c" "src/sma/smaRollup.c" "src/sma/smaTimeRange.c" diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 902e5e1bcc..a4233e1ac4 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -34,7 +34,8 @@ extern "C" { typedef struct SSmaEnv SSmaEnv; typedef struct SSmaStat SSmaStat; -typedef struct SSmaStatItem SSmaStatItem; +typedef struct STSmaStat STSmaStat; +typedef struct SRSmaStat SRSmaStat; typedef struct SSmaKey SSmaKey; typedef struct SRSmaInfo SRSmaInfo; typedef struct SRSmaInfoItem SRSmaInfoItem; @@ -45,26 +46,38 @@ struct SSmaEnv { SSmaStat *pStat; }; -#define SMA_ENV_LOCK(env) ((env)->lock) -#define SMA_ENV_TYPE(env) ((env)->type) -#define SMA_ENV_STAT(env) ((env)->pStat) -#define SMA_ENV_STAT_ITEM(env) ((env)->pStat->tsmaStatItem) +#define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_TYPE(env) ((env)->type) +#define SMA_ENV_STAT(env) ((env)->pStat) -struct SSmaStatItem { +struct STSmaStat { int8_t state; // ETsdbSmaStat STSma *pTSma; // cache schema STSchema *pTSchema; }; +struct SRSmaStat { + SSma *pSma; + void *tmrHandle; + tmr_h tmrId; + int8_t tmrStat; + int32_t tmrSeconds; + SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; +}; + struct SSmaStat { union { - SSmaStatItem tsmaStatItem; - SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; + STSmaStat tsmaStat; // time-range-wise sma + SRSmaStat rsmaStat; // rollup sma }; T_REF_DECLARE() }; -#define SMA_STAT_ITEM(s) ((s)->tsmaStatItem) -#define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash) +#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) +#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) +#define SMA_RSMA_INFO_HASH(s) ((s)->rsmaStat.rsmaInfoHash) +#define SMA_RSMA_TMR_HANDLE(s) ((s)->rsmaStat.tmrHandle) +#define SMA_RSMA_TMR_STAT(s) ((s)->rsmaStat.tmrStat) +#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -107,53 +120,51 @@ static FORCE_INLINE int32_t tdUnLockSmaEnv(SSmaEnv *pEnv) { return 0; } -static FORCE_INLINE int8_t tdSmaStat(SSmaStatItem *pStatItem) { - if (pStatItem) { - return atomic_load_8(&pStatItem->state); +static FORCE_INLINE int8_t tdSmaStat(STSmaStat *pTStat) { + if (pTStat) { + return atomic_load_8(&pTStat->state); } return TSDB_SMA_STAT_UNKNOWN; } -static FORCE_INLINE bool tdSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) { - if (!pStatItem) { +static FORCE_INLINE bool tdSmaStatIsOK(STSmaStat *pTStat, int8_t *state) { + if (!pTStat) { return false; } if (state) { - *state = atomic_load_8(&pStatItem->state); + *state = atomic_load_8(&pTStat->state); return *state == TSDB_SMA_STAT_OK; } - return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK; + return atomic_load_8(&pTStat->state) == TSDB_SMA_STAT_OK; } -static FORCE_INLINE bool tdSmaStatIsExpired(SSmaStatItem *pStatItem) { - return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true; +static FORCE_INLINE bool tdSmaStatIsExpired(STSmaStat *pTStat) { + return pTStat ? (atomic_load_8(&pTStat->state) & TSDB_SMA_STAT_EXPIRED) : true; } -static FORCE_INLINE bool tdSmaStatIsDropped(SSmaStatItem *pStatItem) { - return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true; +static FORCE_INLINE bool tdSmaStatIsDropped(STSmaStat *pTStat) { + return pTStat ? (atomic_load_8(&pTStat->state) & TSDB_SMA_STAT_DROPPED) : true; } -static FORCE_INLINE void tdSmaStatSetOK(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK); +static FORCE_INLINE void tdSmaStatSetOK(STSmaStat *pTStat) { + if (pTStat) { + atomic_store_8(&pTStat->state, TSDB_SMA_STAT_OK); } } -static FORCE_INLINE void tdSmaStatSetExpired(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED); +static FORCE_INLINE void tdSmaStatSetExpired(STSmaStat *pTStat) { + if (pTStat) { + atomic_or_fetch_8(&pTStat->state, TSDB_SMA_STAT_EXPIRED); } } -static FORCE_INLINE void tdSmaStatSetDropped(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED); +static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { + if (pTStat) { + atomic_or_fetch_8(&pTStat->state, TSDB_SMA_STAT_DROPPED); } } -static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType); -void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); @@ -163,6 +174,50 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); +typedef struct STFInfo STFInfo; +typedef struct STFile STFile; + +struct STFInfo { + uint32_t magic; + uint32_t fver; + uint64_t size; +}; + +struct STFile { + STFInfo info; + STfsFile f; + TdFilePtr pFile; + uint8_t state; +}; + +#define TD_FILE_F(tf) (&((tf)->f)) +#define TD_FILE_PFILE(tf) ((tf)->pFile) +#define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL) +#define TD_FILE_FULL_NAME(tf) (TD_FILE_F(tf)->aname) +#define TD_FILE_REL_NAME(tf) (TD_FILE_F(tf)->rname) +#define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL) +#define TD_FILE_CLOSED(tf) (!TD_FILE_OPENED(tf)) +#define TD_FILE_SET_CLOSED(f) (TD_FILE_PFILE(f) = NULL) +#define TD_FILE_STATE(tf) ((tf)->state) +#define TD_FILE_SET_STATE(tf, s) ((tf)->state = (s)) +#define TD_FILE_DID(tf) (TD_FILE_F(tf)->did) +#define TD_FILE_IS_OK(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_OK) +#define TD_FILE_IS_BAD(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_BAD) + +int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname); +int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType); +int32_t tdOpenTFile(STFile *pTFile, int flags); +int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte); +int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence); +int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte); +int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset); +int32_t tdRemoveTFile(STFile *pTFile); +int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo); +int32_t tdUpdateTFileHeader(STFile *pTFile); +void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); +void tdCloseTFile(STFile *pTFile); +void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index a5ca90e55f..2eb39cdddf 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -265,8 +265,8 @@ struct SMCtbCursor { SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { SMCtbCursor *pCtbCur = NULL; SCtbIdxKey ctbIdxKey; - int ret; - int c; + int ret = 0; + int c = 0; pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); if (pCtbCur == NULL) { diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index a80af2b202..4e907e93ca 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -21,9 +21,10 @@ typedef struct SSmaStat SSmaStat; // declaration of static functions -static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType); +static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma); static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path); static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaEnv **pEnv); +static void *tdFreeTSmaStat(STSmaStat *pStat); // implementation @@ -45,7 +46,7 @@ static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path) return NULL; } - if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) { + if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma) != TSDB_CODE_SUCCESS) { tdFreeSmaEnv(pEnv); return NULL; } @@ -105,7 +106,7 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { return 0; } -static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { +static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) { ASSERT(pSmaStat != NULL); if (*pSmaStat) { // no lock @@ -125,10 +126,23 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { } if (smaType == TSDB_SMA_TYPE_ROLLUP) { - SMA_STAT_INFO_HASH(*pSmaStat) = taosHashInit( - RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + SMA_RSMA_STAT(*pSmaStat)->pSma = (SSma*)pSma; + // init timer + SMA_RSMA_TMR_HANDLE(*pSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA_G"); + if (!SMA_RSMA_TMR_HANDLE(*pSmaStat)) { + taosMemoryFreeClear(*pSmaStat); + return TSDB_CODE_FAILED; + } + + atomic_store_8(&SMA_RSMA_TMR_STAT(*pSmaStat), TASK_TRIGGER_STATUS__ACTIVE); - if (!SMA_STAT_INFO_HASH(*pSmaStat)) { + // init hash + SMA_RSMA_INFO_HASH(*pSmaStat) = taosHashInit( + RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + if (!SMA_RSMA_INFO_HASH(*pSmaStat)) { + if (SMA_RSMA_TMR_HANDLE(*pSmaStat)) { + taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(*pSmaStat)); + } taosMemoryFreeClear(*pSmaStat); return TSDB_CODE_FAILED; } @@ -141,16 +155,16 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { return TSDB_CODE_SUCCESS; } -void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { - if (pSmaStatItem) { - tDestroyTSma(pSmaStatItem->pTSma); - taosMemoryFreeClear(pSmaStatItem->pTSma); - taosMemoryFreeClear(pSmaStatItem); +static void *tdFreeTSmaStat(STSmaStat *pStat) { + if (pStat) { + tDestroyTSma(pStat->pTSma); + taosMemoryFreeClear(pStat->pTSma); + taosMemoryFreeClear(pStat); } return NULL; } -void* tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { +void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { tdDestroySmaState(pSmaStat, smaType); taosMemoryFreeClear(pSmaStat); return NULL; @@ -165,16 +179,19 @@ void* tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (pSmaStat) { if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - tdFreeSmaStatItem(&pSmaStat->tsmaStatItem); + tdFreeTSmaStat(&pSmaStat->tsmaStat); } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { + if (SMA_RSMA_TMR_HANDLE(pSmaStat)) { + taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(pSmaStat)); + } // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. - void *infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), NULL); + void *infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), NULL); while (infoHash) { SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash; tdFreeRSmaInfo(pInfoHash); - infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), infoHash); + infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), infoHash); } - taosHashCleanup(SMA_STAT_INFO_HASH(pSmaStat)); + taosHashCleanup(SMA_RSMA_INFO_HASH(pSmaStat)); } else { ASSERT(0); } @@ -273,4 +290,4 @@ void smaTimerCleanUp(void *timer, int8_t *initFlag) { taosTmrCleanUp(timer); atomic_store_8(initFlag, 0); } -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index a1c47a96c0..383c264e7b 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -137,4 +137,17 @@ int32_t smaClose(SSma *pSma) { taosMemoryFreeClear(pSma); } return 0; +} + +/** + * @brief rsma env restore + * + * @param pSma + * @return int32_t + */ +int32_t smaRestore(SSma *pSma) { + if (!pSma) return 0; + // iterate all stables to restore the rsma env + + return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index f70c98bae6..8576c9ca43 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -16,12 +16,17 @@ #include "sma.h" #include "tstream.h" +typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T; +static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; + static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle, int8_t idx); static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem, tb_uid_t suid, int8_t level); +static void tdRSmaFetchTrigger(void *param, void *tmrId); +static void tdRSmaPersistTrigger(void *param, void *tmrId); struct SRSmaInfoItem { SRSmaInfo *pRsmaInfo; @@ -33,14 +38,6 @@ struct SRSmaInfoItem { int8_t triggerStatus; // TASK_TRIGGER_STATUS__IN_ACTIVE/TASK_TRIGGER_STATUS__ACTIVE int32_t maxDelay; }; - -typedef struct { - int64_t suid; - SRSmaInfoItem *pItem; - SSma *pSma; - STSchema *pTSchema; -} SRSmaTriggerParam; - struct SRSmaInfo { STSchema *pTSchema; SSma *pSma; @@ -95,7 +92,7 @@ static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SA return TSDB_CODE_FAILED; } - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); terrno = TSDB_CODE_RSMA_INVALID_STAT; @@ -164,7 +161,7 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui SSmaStat *pStat = SMA_ENV_STAT(pEnv); SHashObj *infoHash = NULL; - if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) { + if (!pStat || !(infoHash = SMA_RSMA_INFO_HASH(pStat))) { terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } @@ -257,7 +254,7 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); if (pRSmaInfo) { ASSERT(0); // TODO: free original pRSmaInfo is exists abnormally smaWarn("vgId:%d, rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); @@ -300,7 +297,7 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { goto _err; } - if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { + if (taosHashPut(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { goto _err; } else { smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid); @@ -449,8 +446,9 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) if (!output) { break; } + if (!pResult) { - pResult = taosArrayInit(0, sizeof(SSDataBlock)); + pResult = taosArrayInit(1, sizeof(SSDataBlock)); if (!pResult) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; @@ -461,7 +459,7 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) } if (taosArrayGetSize(pResult) > 0) { -#if 0 +#if 1 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, pItem->level); blockDebugShowData(pResult, flag); @@ -469,14 +467,12 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); SSubmitReq *pReq = NULL; if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) { - taosArrayDestroy(pResult); - return TSDB_CODE_FAILED; + goto _err; } if (pReq && tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) < 0) { - taosArrayDestroy(pResult); taosMemoryFreeClear(pReq); - return TSDB_CODE_FAILED; + goto _err; } taosMemoryFreeClear(pReq); @@ -489,7 +485,10 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) } taosArrayDestroy(pResult); - return 0; + return TSDB_CODE_SUCCESS; +_err: + taosArrayDestroy(pResult); + return TSDB_CODE_FAILED; } /** @@ -498,13 +497,12 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) * @param param * @param tmrId */ -static void rsmaTriggerByTimer(void *param, void *tmrId) { - // SRSmaTriggerParam *pParam = (SRSmaTriggerParam *)param; - // SRSmaInfoItem *pItem = pParam->pItem; +static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfoItem *pItem = param; if (atomic_load_8(&pItem->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { - smaTrace("level %" PRIi8 " status is active for tb suid:%" PRIi64, pItem->level, pItem->pRsmaInfo->suid); + smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is active for tb suid:%" PRIi64, __func__, __LINE__, + taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid); SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE); @@ -512,10 +510,11 @@ static void rsmaTriggerByTimer(void *param, void *tmrId) { tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK); } else { - smaTrace("level %" PRIi8 " status is inactive for tb suid:%" PRIi64, pItem->level, pItem->pRsmaInfo->suid); + smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is inactive for tb suid:%" PRIi64, __func__, __LINE__, + taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid); } - // taosTmrReset(rsmaTriggerByTimer, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); + // taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); } static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, @@ -528,16 +527,20 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, pItem->taskInfo, suid); - // inputType = STREAM_DATA_TYPE_SUBMIT_BLOCK(1) - if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) { + if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) { // STREAM_DATA_TYPE_SUBMIT_BLOCK smaError("vgId:%d, rsma % " PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } - // SRSmaTriggerParam triggerParam = {.suid = suid, .pItem = pItem, .pSma = pSma, .pTSchema = pTSchema}; tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK); atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE); - taosTmrReset(rsmaTriggerByTimer, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); + smaWarn("%s:%d THREAD:%" PRIi64 " process rsma insert", __func__, __LINE__, taosGetSelfPthreadId()); + + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat); + + taosTmrStart(tdRSmaPersistTrigger, 5000, pStat, pStat->tmrHandle); + taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); return TSDB_CODE_SUCCESS; } @@ -552,7 +555,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); @@ -604,3 +607,106 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { } return TSDB_CODE_SUCCESS; } + +void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char* outputName) { + tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName); +} + +static void *tdRSmaPersistExec(void *param) { + setThreadName("rsma-task-persist"); + SRSmaStat *pRSmaStat = param; + SSma *pSma = pRSmaStat->pSma; + STfs *pTfs = pSma->pVnode->pTfs; + int64_t toffset = 0; + + void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); + if (!infoHash) { + goto _end; + } + + STFile tFile = {0}; + int32_t vid = 2; + char qTaskInfoFName[TSDB_FILENAME_LEN]; + tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName); + tdInitTFile(&tFile, pTfs, qTaskInfoFName); + tdCreateTFile(&tFile, pTfs, true, -1); + + while (infoHash) { + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; + char *pOutput = NULL; + int32_t len = 0; + if (qSerializeTaskStatus(pRSmaInfo->items[0].taskInfo, &pOutput, &len) < 0) { + smaError("serialize rsma task for table %" PRIi64 " failed since %s", pRSmaInfo->items[0].pRsmaInfo->suid, + terrstr(terrno)); + } else { + smaWarn("serialize rsma task for table %" PRIi64 " success and len is %d", pRSmaInfo->items[0].pRsmaInfo->suid, + len); + } + tdAppendTFile(&tFile, &len, sizeof(len), &toffset); + tdAppendTFile(&tFile, pOutput, len, &toffset); + + taosMemoryFree(pOutput); + infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash); + } +_end: + + if (tdUpdateTFileHeader(&tFile) < 0) { + smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno)); + tdCloseTFile(&tFile); + tdRemoveTFile(&tFile); + return NULL; + } + + tdCloseTFile(&tFile); + + char newFName[TSDB_FILENAME_LEN]; + strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); + char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]); + strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); + taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName); + + atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE); + return NULL; +_err: + atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE); + // remove the .tmp file + return NULL; +} + +static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { + smaWarn("%s:%d entry ", __func__, __LINE__); + TdThread threadId; + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); + + if (taosThreadCreate(&threadId, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) { + smaError("failed to create thread to persist rsma qTaskInfo since %s", strerror(errno)); + } + + taosThreadAttrDestroy(&thAttr); + smaWarn("%s:%d end ", __func__, __LINE__); +} + +/** + * @brief trigger to persist rsma qTaskInfo + * + * @param param + * @param tmrId + */ +static void tdRSmaPersistTrigger(void *param, void *tmrId) { + SRSmaStat *pRSmaStat = param; + + if (atomic_load_8(&pRSmaStat->tmrStat) == TASK_TRIGGER_STATUS__ACTIVE) { + smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence start since active", __func__, __LINE__, taosGetSelfPthreadId()); + atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__IN_ACTIVE); + + // execution + tdRSmaPersistTask(pRSmaStat); + } else { + smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence not start since inactive", __func__, __LINE__, + taosGetSelfPthreadId()); + } + + taosTmrReset(tdRSmaPersistTrigger, 5000, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId); +} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 4352c466c5..2244b91c28 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -129,7 +129,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); SSmaStat *pStat = NULL; - SSmaStatItem *pItem = NULL; + STSmaStat *pItem = NULL; if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) { terrno = TSDB_CODE_TSMA_INVALID_STAT; @@ -137,7 +137,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { } tdRefSmaStat(pSma, pStat); - pItem = &pStat->tsmaStatItem; + pItem = &pStat->tsmaStat; ASSERT(pItem); diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c new file mode 100644 index 0000000000..77e85d3227 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" + +#define TD_FILE_HEAD_SIZE 512 + +#define TD_FILE_STATE_OK 0 +#define TD_FILE_STATE_BAD 1 + +#define TD_FILE_INIT_MAGIC 0xFFFFFFFF + + +static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo); +static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo); + +static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) { + int32_t tlen = 0; + + tlen += taosEncodeFixedU32(buf, pInfo->magic); + tlen += taosEncodeFixedU32(buf, pInfo->fver); + tlen += taosEncodeFixedU64(buf, pInfo->size); + + return tlen; +} + +static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) { + buf = taosDecodeFixedU32(buf, &(pInfo->magic)); + buf = taosDecodeFixedU32(buf, &(pInfo->fver)); + buf = taosDecodeFixedU64(buf, &(pInfo->size)); + return buf; +} + +int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte) { + ASSERT(TD_FILE_OPENED(pTFile)); + + int64_t nwrite = taosWriteFile(pTFile->pFile, buf, nbyte); + if (nwrite < nbyte) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nwrite; +} + +int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) { + ASSERT(TD_FILE_OPENED(pTFile)); + + int64_t loffset = taosLSeekFile(TD_FILE_PFILE(pTFile), offset, whence); + if (loffset < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return loffset; +} + +int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) { + ASSERT(TD_FILE_OPENED(pTFile)); + + int64_t nread = taosReadFile(pTFile->pFile, buf, nbyte); + if (nread < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nread; +} + +int32_t tdUpdateTFileHeader(STFile *pTFile) { + char buf[TD_FILE_HEAD_SIZE] = "\0"; + + if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) { + return -1; + } + + void *ptr = buf; + tdEncodeTFInfo(&ptr, &(pTFile->info)); + + taosCalcChecksumAppend(0, (uint8_t *)buf, TD_FILE_HEAD_SIZE); + if (tdWriteTFile(pTFile, buf, TD_FILE_HEAD_SIZE) < 0) { + return -1; + } + + return 0; +} + +int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo) { + char buf[TD_FILE_HEAD_SIZE] = "\0"; + uint32_t _version; + + ASSERT(TD_FILE_OPENED(pTFile)); + + if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) { + return -1; + } + + if (tdReadTFile(pTFile, buf, TD_FILE_HEAD_SIZE) < 0) { + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)buf, TD_FILE_HEAD_SIZE)) { + terrno = TSDB_CODE_FILE_CORRUPTED; + return -1; + } + + void *pBuf = buf; + pBuf = tdDecodeTFInfo(pBuf, pInfo); + return 0; +} + +void tdUpdateTFileMagic(STFile *pTFile, void *pCksm) { + pTFile->info.magic = taosCalcChecksum(pTFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM)); +} + +int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) { + ASSERT(TD_FILE_OPENED(pTFile)); + + int64_t toffset; + + if ((toffset = tdSeekTFile(pTFile, 0, SEEK_END)) < 0) { + return -1; + } + + ASSERT(pTFile->info.size == toffset); + + if (offset) { + *offset = toffset; + } + + if (tdWriteTFile(pTFile, buf, nbyte) < 0) { + return -1; + } + + pTFile->info.size += nbyte; + + return nbyte; +} + +int32_t tdOpenTFile(STFile *pTFile, int flags) { + ASSERT(!TD_FILE_OPENED(pTFile)); + + pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), flags); + if (pTFile->pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} + +void tdCloseTFile(STFile *pTFile) { + if (TD_FILE_OPENED(pTFile)) { + taosCloseFile(&pTFile->pFile); + TD_FILE_SET_CLOSED(pTFile); + } +} + +void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName) { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vid, dname, fname); +} + +int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) { + char fullname[TSDB_FILENAME_LEN]; + SDiskID did = {0}; + + TD_FILE_SET_STATE(pTFile, TD_FILE_STATE_OK); + TD_FILE_SET_CLOSED(pTFile); + + memset(&(pTFile->info), 0, sizeof(pTFile->info)); + pTFile->info.magic = TD_FILE_INIT_MAGIC; + + if (tfsAllocDisk(pTfs, 0, &did) < 0) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return -1; + } + + tfsInitFile(pTfs, &(pTFile->f), did, fname); + + return 0; +} + +int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType) { + ASSERT(pTFile->info.size == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC); + + pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pTFile->pFile == NULL) { + if (errno == ENOENT) { + // Try to create directory recursively + char *s = strdup(TD_FILE_REL_NAME(pTFile)); + if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_FILE_DID(pTFile)) < 0) { + taosMemoryFreeClear(s); + return -1; + } + taosMemoryFreeClear(s); + + pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pTFile->pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } else { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } + + if (!updateHeader) { + return 0; + } + + pTFile->info.size += TD_FILE_HEAD_SIZE; + pTFile->info.fver = 0; + + if (tdUpdateTFileHeader(pTFile) < 0) { + tdCloseTFile(pTFile); + tdRemoveTFile(pTFile); + return -1; + } + + return 0; +} + +int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); } \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b89e908df7..fa10fc26dd 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -76,6 +76,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NEED_RETRY, "Retry needed") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE, "Out of memory in rpc queue") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIMESTAMP, "Invalid timestamp format") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR, "Msg decode error") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") @@ -356,7 +357,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, "No table data in memo TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS, "File already exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE, "Need to reconfigure table") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO, "Invalid information to create table") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK, "No available disk") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK, "TSDB no available disk") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG, "TSDB messed message") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag value") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") From 95c1b0005610d033b0d524c2c3ace9a0fafcd754 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 23 Jun 2022 14:25:30 +0800 Subject: [PATCH 33/54] enh: modify delete output res --- include/common/tcommon.h | 1 + include/libs/executor/dataSinkMgt.h | 5 +++-- include/libs/qworker/qworker.h | 2 +- source/libs/executor/src/dataDeleter.c | 2 +- source/libs/executor/src/executil.c | 2 ++ source/libs/executor/src/executorimpl.c | 1 + source/libs/qworker/src/qworker.c | 2 +- tests/script/tsim/show/basic.sim | 2 +- 8 files changed, 11 insertions(+), 6 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5dac79551a..5c102dc30a 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -54,6 +54,7 @@ typedef struct { SHashObj* map; // speedup acquire the tableQueryInfo by table uid void* pTagCond; void* pTagIndexCond; + uint64_t suid; } STableListInfo; typedef struct SColumnDataAgg { diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index c23cf162aa..957c40f21e 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -33,7 +33,7 @@ struct SDataSink; struct SSDataBlock; typedef struct SDeleterRes { - uint64_t uid; + uint64_t suid; SArray* uidList; int64_t skey; int64_t ekey; @@ -41,7 +41,8 @@ typedef struct SDeleterRes { } SDeleterRes; typedef struct SDeleterParam { - SArray* pUidList; + uint64_t suid; + SArray* pUidList; } SDeleterParam; typedef struct SDataSinkStat { diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 3b8a37f420..36e9b3309c 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -32,7 +32,7 @@ enum { }; typedef struct SDeleteRes { - uint64_t uid; + uint64_t suid; SArray* uidList; int64_t skey; int64_t ekey; diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 33b7811e6c..9db2783ccd 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -86,7 +86,7 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0); SDeleterRes* pRes = (SDeleterRes*)pEntry->data; - pRes->uid = pHandle->pDeleter->tableId; + pRes->suid = pHandle->pParam->suid; pRes->uidList = pHandle->pParam->pUidList; pRes->skey = pHandle->pDeleter->deleteTimeRange.skey; pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 0282d9cccb..ac38fd8de9 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -300,6 +300,8 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo uint64_t tableUid = pScanNode->uid; + pListInfo->suid = pScanNode->suid; + SNode* pTagCond = (SNode*)pListInfo->pTagCond; SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond; if (pScanNode->tableType == TSDB_SUPER_TABLE) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 30ed44b872..8c9a6ac3a5 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4464,6 +4464,7 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT return TSDB_CODE_OUT_OF_MEMORY; } int32_t tbNum = taosArrayGetSize(pTask->tableqinfoList.pTableList); + pDeleterParam->suid = pTask->tableqinfoList.suid; pDeleterParam->pUidList = taosArrayInit(tbNum, sizeof(uint64_t)); if (NULL == pDeleterParam->pUidList) { taosMemoryFree(pDeleterParam); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 28181d7e11..4250785f7c 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -271,7 +271,7 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen SDeleterRes* pDelRes = (SDeleterRes*)output.pData; rsp.affectedRows = pDelRes->affectedRows; - pRes->uid = pDelRes->uid; + pRes->suid = pDelRes->suid; pRes->uidList = pDelRes->uidList; pRes->skey = pDelRes->skey; pRes->ekey = pDelRes->ekey; diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index 8f9362fc69..e2138e0008 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -233,7 +233,7 @@ if $rows != 4 then endi sql show dnode 1 variables; -if $rows != 114 then +if $rows != 116 then return -1 endi From 40ed0e22bf2255872e529bfcbf6c049637196df3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 14:28:32 +0800 Subject: [PATCH 34/54] test: minor changes --- tests/script/tsim/mnode/basic5.sim | 1 + tests/script/tsim/show/basic.sim | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index 23f5f6d782..6121ecb912 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -233,6 +233,7 @@ if $data(1)[3] != dropping then endi print =============== step9: start mnode1 and wait it dropped +sleep 1000 system sh/exec.sh -n dnode1 -s start $x = 0 diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index 8f9362fc69..4d646f39e3 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -233,12 +233,12 @@ if $rows != 4 then endi sql show dnode 1 variables; -if $rows != 114 then +if $rows <= 0 then return -1 endi sql show local variables; -if $rows != 50 then +if $rows <= 0 then return -1 endi From b434ec8cd8f9aff704eb9780f5c26dfc514ec41c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 14:30:35 +0800 Subject: [PATCH 35/54] test: minor changes --- tests/script/tsim/mnode/basic5.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index 6121ecb912..e298dcbe55 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -233,7 +233,7 @@ if $data(1)[3] != dropping then endi print =============== step9: start mnode1 and wait it dropped -sleep 1000 +sleep 3000 system sh/exec.sh -n dnode1 -s start $x = 0 From 147a984c17631d29392bb346d4670d7db6ad6d7c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 15:11:23 +0800 Subject: [PATCH 36/54] feat: make config dnode work --- source/common/src/tglobal.c | 11 ++++++++--- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 11 +++++++++-- source/util/src/tlog.c | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 6cae3a13e6..c05d4cf1be 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -760,9 +760,14 @@ void taosCleanupCfg() { } void taosCfgDynamicOptions(const char *option, const char *value) { - if (strcasecmp(option, "debugFlag") == 0) { - int32_t debugFlag = atoi(value); - taosSetAllDebugFlag(debugFlag); + if (strncasecmp(option, "debugFlag", 9) == 0) { + if (value != NULL) { + if (strlen(option) > 10) { + value = option + 10; + } + } + int32_t flag = atoi(value); + taosSetAllDebugFlag(flag); } if (strcasecmp(option, "resetlog") == 0) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 9c8918a445..44b6f0ee85 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -123,8 +123,15 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { - dError("config req is received, but not supported yet"); - return TSDB_CODE_OPS_NOT_SUPPORT; + SDCfgDnodeReq cfgReq = {0}; + if (tDeserializeSMCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); + taosCfgDynamicOptions(cfgReq.config, cfgReq.value); + return 0; } static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 353e94a490..0439ed148b 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -743,7 +743,7 @@ cmp_end: } void taosSetAllDebugFlag(int32_t flag) { - if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return; + if (flag <= 0) return; dDebugFlag = flag; vDebugFlag = flag; From b2f81331a558c817e7523bd3b4e631339ee96b7d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 23 Jun 2022 15:28:02 +0800 Subject: [PATCH 37/54] other: code optimization for rsma --- source/dnode/vnode/src/inc/sma.h | 3 ++- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/sma/smaUtil.c | 14 ++++++++------ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index a4233e1ac4..1489b02492 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -179,8 +179,9 @@ typedef struct STFile STFile; struct STFInfo { uint32_t magic; + uint32_t ftype; uint32_t fver; - uint64_t size; + uint64_t fsize; }; struct STFile { diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 8576c9ca43..3c2f710fa7 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -708,5 +708,5 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) { taosGetSelfPthreadId()); } - taosTmrReset(tdRSmaPersistTrigger, 5000, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId); + taosTmrReset(tdRSmaPersistTrigger, 3600000, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId); } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 77e85d3227..76690a3bde 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -30,16 +30,18 @@ static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) { int32_t tlen = 0; tlen += taosEncodeFixedU32(buf, pInfo->magic); + tlen += taosEncodeFixedU32(buf, pInfo->ftype); tlen += taosEncodeFixedU32(buf, pInfo->fver); - tlen += taosEncodeFixedU64(buf, pInfo->size); + tlen += taosEncodeFixedU64(buf, pInfo->fsize); return tlen; } static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->magic)); + buf = taosDecodeFixedU32(buf, &(pInfo->ftype)); buf = taosDecodeFixedU32(buf, &(pInfo->fver)); - buf = taosDecodeFixedU64(buf, &(pInfo->size)); + buf = taosDecodeFixedU64(buf, &(pInfo->fsize)); return buf; } @@ -134,7 +136,7 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) return -1; } - ASSERT(pTFile->info.size == toffset); + ASSERT(pTFile->info.fsize == toffset); if (offset) { *offset = toffset; @@ -144,7 +146,7 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) return -1; } - pTFile->info.size += nbyte; + pTFile->info.fsize += nbyte; return nbyte; } @@ -193,7 +195,7 @@ int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) { } int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType) { - ASSERT(pTFile->info.size == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC); + ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC); pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pTFile->pFile == NULL) { @@ -221,7 +223,7 @@ int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fTyp return 0; } - pTFile->info.size += TD_FILE_HEAD_SIZE; + pTFile->info.fsize += TD_FILE_HEAD_SIZE; pTFile->info.fver = 0; if (tdUpdateTFileHeader(pTFile) < 0) { From cbacd1fd4578570836657a8332478dfad5f2e0a0 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 15:35:19 +0800 Subject: [PATCH 38/54] fix: fix empty downstream input for group sort operator --- source/libs/executor/src/sortoperator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 6086138301..24016c3338 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -387,6 +387,10 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { pInfo->hasGroupId = true; pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]); + if (pInfo->prefetchedSortInput == NULL) { + doSetOperatorCompleted(pOperator); + return NULL; + } pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId; pInfo->childOpStatus = CHILD_OP_NEW_GROUP; beginSortGroup(pOperator); From 0811e5fc93416e883632a9df2ac30ba3dc6ce054 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 15:35:22 +0800 Subject: [PATCH 39/54] feat: make config dnode work --- include/common/tmsg.h | 10 ++++- source/common/src/tglobal.c | 5 --- source/common/src/tmsg.c | 27 +++++++++++++ source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 43 +++++++++++++++------ 5 files changed, 69 insertions(+), 18 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0ef9dffa2e..c32b7be941 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1346,11 +1346,19 @@ typedef struct { int32_t dnodeId; char config[TSDB_DNODE_CONFIG_LEN]; char value[TSDB_DNODE_VALUE_LEN]; -} SMCfgDnodeReq, SDCfgDnodeReq; +} SMCfgDnodeReq; int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); +typedef struct { + char config[TSDB_DNODE_CONFIG_LEN]; + char value[TSDB_DNODE_VALUE_LEN]; +} SDCfgDnodeReq; + +int32_t tSerializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq); +int32_t tDeserializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq); + typedef struct { int32_t dnodeId; } SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c05d4cf1be..2172a6fe50 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -761,11 +761,6 @@ void taosCleanupCfg() { void taosCfgDynamicOptions(const char *option, const char *value) { if (strncasecmp(option, "debugFlag", 9) == 0) { - if (value != NULL) { - if (strlen(option) > 10) { - value = option + 10; - } - } int32_t flag = atoi(value); taosSetAllDebugFlag(flag); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6bd0dc02e1..4ad11f2998 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1500,6 +1500,33 @@ int32_t tDeserializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq return 0; } +int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->config) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->value) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->config) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->value) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 44b6f0ee85..1ffd86beb1 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -124,7 +124,7 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { SDCfgDnodeReq cfgReq = {0}; - if (tDeserializeSMCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { + if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index b159061ce3..d3723326b9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -502,20 +502,20 @@ _OVER: } static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - SDnodeObj *pObj = NULL; - void *pIter = NULL; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + SDnodeObj *pObj = NULL; + void *pIter = NULL; SDnodeListRsp rsp = {0}; - int32_t code = -1; - + int32_t code = -1; + rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet)); if (NULL == rsp.dnodeList) { mError("failed to alloc epSet while process dnode list req"); terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - + while (1) { pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj); if (pIter == NULL) break; @@ -554,7 +554,6 @@ _OVER: return code; } - static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -732,15 +731,37 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr()); return -1; } - SEpSet epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - int32_t bufLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); + SDCfgDnodeReq dcfgReq = {0}; + if (strncasecmp(cfgReq.config, "debugFlag", 9) == 0) { + const char *value = cfgReq.value; + int32_t flag = atoi(value); + if (flag <= 0) { + flag = atoi(cfgReq.config + 10); + } + if (flag <= 0 || flag > 255) { + mError("dnode:%d, failed to config debugFlag since value:%d", cfgReq.dnodeId, flag); + terrno = TSDB_CODE_INVALID_CFG; + return -1; + } + + strcpy(dcfgReq.config, "debugFlag"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); + } else if (strcasecmp(cfgReq.config, "resetlog") == 0) { + strcpy(dcfgReq.config, "resetlog"); + } else { + terrno = TSDB_CODE_INVALID_CFG; + mError("dnode:%d, failed to config since %s", cfgReq.dnodeId, terrstr()); + return -1; + } + + int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, &dcfgReq); void *pBuf = rpcMallocCont(bufLen); if (pBuf == NULL) return -1; - tSerializeSMCfgDnodeReq(pBuf, bufLen, &cfgReq); + tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq); mDebug("dnode:%d, send config req to dnode, app:%p", cfgReq.dnodeId, pReq->info.ahandle); SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; From 71bda7e6586b243b9acc111398a48dbe37fd22ac Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 23 Jun 2022 15:46:42 +0800 Subject: [PATCH 40/54] feat(tmq): add with meta config --- examples/c/tmq.c | 10 +++++--- source/client/src/clientMain.c | 2 ++ source/client/src/tmq.c | 26 +++++++++---------- source/dnode/vnode/src/tq/tq.c | 1 + source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/scanoperator.c | 33 ++++++++++++++----------- 6 files changed, 41 insertions(+), 33 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 980c86b877..25e4b5c193 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -26,6 +26,10 @@ static void msg_process(TAOS_RES* msg) { printf("topic: %s\n", tmq_get_topic_name(msg)); printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); + if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { + printf("meta, skip\n"); + return; + } while (1) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; @@ -129,8 +133,8 @@ int32_t create_topic() { } taos_free_result(pRes); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as database abc1");*/ - pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); + pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); + /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; @@ -191,7 +195,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); - tmq_conf_set(conf, "experiment.use.snapshot", "true"); + tmq_conf_set(conf, "experiment.use.snapshot", "false"); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index ab4d14b1cc..22747425f0 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -290,6 +290,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { pResultInfo->current += 1; return pResultInfo->row; } + } else if (TD_RES_TMQ_META(res)) { + return NULL; } else { // assert to avoid un-initialization error ASSERT(0); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 949965296f..6892a4d1a7 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1160,8 +1160,6 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { // handle meta rsp int8_t rspType = ((SMqRspHead*)pMsg->pData)->mqMsgType; - if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { - } SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { @@ -1174,19 +1172,19 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { pRspWrapper->vgHandle = pVg; pRspWrapper->topicHandle = pTopic; - memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); - if (rspType == TMQ_MSG_TYPE__POLL_RSP) { + memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp); } else { ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP); + memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqMetaRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->metaRsp); } taosMemoryFree(pMsg->pData); - tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId, - pRspWrapper->dataRsp.reqOffset, pRspWrapper->dataRsp.rspOffset); + tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld, type %d", tmq->consumerId, pVg->vgId, + pRspWrapper->dataRsp.reqOffset, pRspWrapper->dataRsp.rspOffset, rspType); taosWriteQitem(tmq->mqueue, pRspWrapper); tsem_post(&tmq->rspSem); @@ -1558,7 +1556,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); - pRspObj->resType = RES_TYPE__TMQ; + pRspObj->resType = RES_TYPE__TMQ_META; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; @@ -1676,7 +1674,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) return 0; } -SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { +void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { while (1) { SMqRspWrapper* rspWrapper = NULL; taosGetQitem(tmq->qall, (void**)&rspWrapper); @@ -1716,18 +1714,18 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) { + if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ - pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset; + pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // build rsp - SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper); + SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); taosFreeQitem(pollRspWrapper); return pRsp; } else { tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n", - pollRspWrapper->dataRsp.head.epoch, consumerEpoch); + pollRspWrapper->metaRsp.head.epoch, consumerEpoch); taosFreeQitem(pollRspWrapper); } } else { @@ -1744,8 +1742,8 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { - SMqRspObj* rspObj; - int64_t startTime = taosGetTimestampMs(); + void* rspObj; + int64_t startTime = taosGetTimestampMs(); #if 0 tmqHandleAllDelayedTask(tmq); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b23ec13105..b428e9aa9f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -387,6 +387,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pHandle->epoch = -1; pHandle->execHandle.subType = req.subType; + pHandle->fetchMeta = req.withMeta; pHandle->pWalReader = walOpenReadHandle(pTq->pVnode->pWal); for (int32_t i = 0; i < 5; i++) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 286bcea820..91931c2fd8 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -340,7 +340,7 @@ typedef struct SStreamBlockScanInfo { SReadHandle readHandle; uint64_t tableUid; // queried super table uid EStreamScanMode scanMode; - SOperatorInfo* pOperatorDumy; + SOperatorInfo* pSnapshotReadOp; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. SArray* childIds; SessionWindowSupporter sessionSup; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a703be86e4..80dd01cf4c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -337,7 +337,8 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_ } for (int32_t i = 0; i < pBlock->info.rows; ++i) { - colDataAppend(pColInfoData, i, data, (data == NULL) || (pColInfoData->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); + colDataAppend(pColInfoData, i, data, + (data == NULL) || (pColInfoData->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); } if (data && (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) && p != NULL && @@ -776,7 +777,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { if (!needRead) { return false; } - STableScanInfo* pTableScanInfo = pInfo->pOperatorDumy->info; + STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info; pTableScanInfo->cond.twindows[0] = win; pTableScanInfo->curTWinIdx = 0; tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); @@ -821,11 +822,11 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_ static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) { while (1) { SSDataBlock* pResult = NULL; - pResult = doTableScan(pInfo->pOperatorDumy); + pResult = doTableScan(pInfo->pSnapshotReadOp); if (pResult == NULL) { if (prepareDataScan(pInfo)) { // scan next window data - pResult = doTableScan(pInfo->pOperatorDumy); + pResult = doTableScan(pInfo->pSnapshotReadOp); } } if (!pResult) { @@ -860,11 +861,11 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa ASSERT(pBlock->info.numOfCols == pUpdateBlock->info.numOfCols); int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex); - pInfo->groupId = getGroupId(pInfo->pOperatorDumy, pBlock, rowId); + pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId); int32_t i = 0; for (; i < size; i++) { rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, i + pInfo->tsArrayIndex); - uint64_t id = getGroupId(pInfo->pOperatorDumy, pBlock, rowId); + uint64_t id = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId); if (pInfo->groupId != id) { break; } @@ -1063,7 +1064,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; } else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) { - SSDataBlock* pResult = doTableScan(pInfo->pOperatorDumy); + SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp); if (pResult) { return pResult->info.rows > 0 ? pResult : NULL; } @@ -1135,7 +1136,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan } else { pInfo->pUpdateInfo = NULL; } - pInfo->pOperatorDumy = pTableScanDummy; + pInfo->pSnapshotReadOp = pTableScanDummy; pInfo->interval = pSTInfo->interval; pInfo->readHandle = *pHandle; @@ -1557,7 +1558,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { return NULL; } - int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE : TDMT_MND_SYSTABLE_RETRIEVE; + int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE + : TDMT_MND_SYSTABLE_RETRIEVE; pMsgSendInfo->param = pOperator; pMsgSendInfo->msgInfo.pData = buf1; @@ -1843,7 +1845,8 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } else { data = (char*)p; } - colDataAppend(pDst, count, data, (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); + colDataAppend(pDst, count, data, + (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) && data != NULL) { @@ -1896,11 +1899,11 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } - pInfo->pTableList = pTableListInfo; - pInfo->pColMatchInfo = colList; - pInfo->pRes = createResDataBlock(pDescNode); - pInfo->readHandle = *pReadHandle; - pInfo->curPos = 0; + pInfo->pTableList = pTableListInfo; + pInfo->pColMatchInfo = colList; + pInfo->pRes = createResDataBlock(pDescNode); + pInfo->readHandle = *pReadHandle; + pInfo->curPos = 0; pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; From d6ccf5720dcae58ca63dc206259efa4e59339fbc Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 23 Jun 2022 15:43:39 +0800 Subject: [PATCH 41/54] fix(stream): valgrind memory error --- source/libs/executor/src/timewindowoperator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 19f0fd4ea7..6813c63542 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2238,7 +2238,8 @@ static void clearUpdateDataBlock(SSDataBlock* pBlock) { } void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) { - ASSERT(pDest->info.capacity >= pSource->info.rows); + // ASSERT(pDest->info.capacity >= pSource->info.rows); + blockDataEnsureCapacity(pDest, pSource->info.rows); clearUpdateDataBlock(pDest); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex); From e07164961973dc52279ae0c676c23769ffe75fdc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 16:34:43 +0800 Subject: [PATCH 42/54] test: update the unit test. --- include/common/tcommon.h | 2 +- source/common/src/tdatablock.c | 3 +++ .../libs/scalar/test/scalar/scalarTests.cpp | 25 ++++++++++++------- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index e8fddee869..4a113f5822 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -68,7 +68,7 @@ typedef struct SColumnDataAgg { typedef struct SDataBlockInfo { STimeWindow window; - int32_t rows; + int32_t rows; // todo hide this attribute int32_t rowSize; uint64_t uid; // the uid of table, from which current data block comes uint16_t blockId; // block id, generated by physical planner diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 15425c8895..8317c6c79d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1109,6 +1109,9 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo * return TSDB_CODE_SUCCESS; } + // todo temp disable it +// ASSERT(pColumn->info.bytes != 0); + int32_t existedRows = pBlockInfo->rows; if (IS_VAR_DATA_TYPE(pColumn->info.type)) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 3324981531..e49c11c390 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -86,15 +86,17 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s if (newBlock) { SSDataBlock *res = createDataBlock(); - SColumnInfoData idata = {0}; - idata.info = *colInfo; + SColumnInfoData idata = {.info = *colInfo}; + colInfoDataEnsureCapacity(&idata, rows); + blockDataAppendColInfo(res, &idata); - blockDataEnsureCapacity(res, rows); + res->info.capacity = rows; res->info.rows = rows; + SColumnInfoData* p = static_cast(taosArrayGet(res->pDataBlock, 0)); + ASSERT(p->pData != NULL && p->nullbitmap != NULL); taosArrayPush(pBlockList, &res); - *dataBlockId = taosArrayGetSize(pBlockList) - 1; res->info.blockId = *dataBlockId; *slotId = 0; @@ -102,6 +104,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); SColumnInfoData idata = {0}; idata.info = *colInfo; + colInfoDataEnsureCapacity(&idata, rows); blockDataAppendColInfo(res, &idata); *dataBlockId = taosArrayGetSize(pBlockList) - 1; @@ -140,17 +143,18 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in if (NULL == *block) { SSDataBlock *res = createDataBlock(); - res->info.rows = rowNum; for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, i + 1); - int32_t size = idata.info.bytes * rowNum; + SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_INT, 10, i + 1); + colInfoDataEnsureCapacity(&idata, rowNum); blockDataAppendColInfo(res, &idata); } SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3); + colInfoDataEnsureCapacity(&idata, rowNum); blockDataAppendColInfo(res, &idata); - blockDataEnsureCapacity(res, rowNum); + res->info.capacity = rowNum; + res->info.rows = rowNum; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -170,8 +174,10 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in int32_t idx = taosArrayGetSize(res->pDataBlock); SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1 + idx); + colInfoDataEnsureCapacity(&idata, rowNum); + + res->info.capacity = rowNum; blockDataAppendColInfo(res, &idata); - blockDataEnsureCapacity(res, rowNum); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); @@ -1507,6 +1513,7 @@ TEST(columnTest, bigint_column_multi_binary_column) { SArray *blockList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(blockList, &src); + SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); int16_t dataBlockId = 0, slotId = 0; scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); From ef00464f9c1b8c9ba7fd4e5b156c384d5cc6c308 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 16:35:05 +0800 Subject: [PATCH 43/54] fix: retry 12 seconds if lock file failed --- source/dnode/mgmt/node_util/src/dmFile.c | 16 +++++++++++++--- tests/script/tsim/mnode/basic5.sim | 1 - 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 78e706f908..2185adc18b 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -127,14 +127,24 @@ TdFilePtr dmCheckRunning(const char *dataDir) { return NULL; } - int32_t ret = taosLockFile(pFile); - if (ret != 0) { + int32_t retryTimes = 0; + int32_t ret = 0; + do { + ret = taosLockFile(pFile); + if (ret == 0) break; + terrno = TAOS_SYSTEM_ERROR(errno); + taosMsleep(100); + retryTimes++; + dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes); + } while (retryTimes < 120); + + if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s", filepath, terrstr()); taosCloseFile(&pFile); return NULL; } + terrno = 0; dDebug("lock file:%s to prevent repeated starts", filepath); return pFile; } diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index e298dcbe55..23f5f6d782 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -233,7 +233,6 @@ if $data(1)[3] != dropping then endi print =============== step9: start mnode1 and wait it dropped -sleep 3000 system sh/exec.sh -n dnode1 -s start $x = 0 From b556d1fa62746d95c988a8dcc8804922d76d3df7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 16:37:36 +0800 Subject: [PATCH 44/54] fix(query): fix to_ISO8601 function crash when processing constant illegal inputs TD-15211 --- source/libs/function/src/builtins.c | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6baf45a5b3..f6d2fdadae 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1267,6 +1267,29 @@ static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) { return false; } +static bool validateTimestampDigits(const SValueNode* pVal) { + if (IS_INTEGER_TYPE(pVal->node.resType.type)) { + return false; + } + + int64_t tsVal = pVal->datum.i; + char fraction[20] = {0}; + NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction); + int32_t tsDigits = (int32_t)strlen(fraction); + + if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) { + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || + tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || + tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + return true; + } else { + return false; + } + } + + return true; +} + static bool validateTimezoneFormat(const SValueNode* pVal) { if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) { return false; @@ -1379,6 +1402,15 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0))) { + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); + + if (!validateTimestampDigits(pValue)) { + pFunc->node.resType = (SDataType){.bytes = 0, .type = TSDB_DATA_TYPE_BINARY}; + return TSDB_CODE_SUCCESS; + } + } + // param1 if (numOfParams == 2) { SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); From c449b7208acdec8656f03af90fce064192b57b9d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 16:52:06 +0800 Subject: [PATCH 45/54] fix: set correct operator blocking and add closing rpc trace to udfd --- source/libs/executor/src/sortoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 2 ++ source/libs/function/src/udfd.c | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 4b5ad7b123..81cf8d6527 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -460,7 +460,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pInfo->pColMatchInfo = pColMatchColInfo; pOperator->name = "GroupSortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT; - pOperator->blocking = true; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->exprSupp.pExprInfo = pExprInfo; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 19f0fd4ea7..7725a939d6 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4117,6 +4117,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI goto _error; } + miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; iaInfo->win = pTaskInfo->window; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 7fffa84e0b..f8713e4b47 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -613,7 +613,9 @@ int32_t udfdOpenClientRpc() { } int32_t udfdCloseClientRpc() { + fnInfo("udfd begin closing rpc"); rpcClose(global.clientRpc); + fnInfo("udfd finish closing rpc"); return 0; } @@ -937,7 +939,6 @@ int main(int argc, char *argv[]) { uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); udfdRun(); - removeListeningPipe(); uv_thread_join(&mnodeConnectThread); udfdCloseClientRpc(); From 14048a5e4c30c4c8ed308342c879d7b8877a58c6 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 23 Jun 2022 17:07:11 +0800 Subject: [PATCH 46/54] fix(tmq): read repeat data after reading snapshot --- source/dnode/vnode/src/tq/tq.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b428e9aa9f..9eec1558b2 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -252,15 +252,17 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { #if 1 if (pReq->useSnapshot) { - tqInfo("retrieve using snapshot"); + // TODO set ver into snapshot int64_t lastVer = walGetCommittedVer(pTq->pWal); if (rsp.reqOffset < lastVer) { + tqInfo("retrieve using snapshot req offset %ld last ver %ld", rsp.reqOffset, lastVer); tqScanSnapshot(pTq, &pHandle->execHandle, &rsp, workerId); if (rsp.blockNum != 0) { rsp.withTbName = false; rsp.rspOffset = lastVer; - tqInfo("direct send by snapshot rsp offset %ld", lastVer); + tqInfo("direct send by snapshot req offset %ld rsp offset %ld", rsp.reqOffset, rsp.rspOffset); + fetchOffset = lastVer; goto SEND_RSP; } } From 323ee6b1b00861a41d6af73746112110e1341257 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 23 Jun 2022 17:26:23 +0800 Subject: [PATCH 47/54] enh(tmq): add demo for tmq_get_raw_meta --- examples/c/tmq.c | 6 +++++- include/client/taos.h | 2 +- source/client/src/tmq.c | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 25e4b5c193..6fb7e7a1fc 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -27,7 +27,11 @@ static void msg_process(TAOS_RES* msg) { printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { - printf("meta, skip\n"); + void* meta; + int32_t metaLen; + tmq_get_raw_meta(msg, &meta, &metaLen); + + printf("meta, len is %d\n", metaLen); return; } while (1) { diff --git a/include/client/taos.h b/include/client/taos.h index 9d4da221f4..d31d5c582c 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -261,7 +261,7 @@ enum tmq_res_t { typedef enum tmq_res_t tmq_res_t; DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); -DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, const void **raw_meta, int32_t *raw_meta_len); +DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, void **raw_meta, int32_t *raw_meta_len); DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 6892a4d1a7..637a7ee5dd 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1871,7 +1871,7 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -int32_t tmq_get_raw_meta(TAOS_RES* res, const void** raw_meta, int32_t* raw_meta_len) { +int32_t tmq_get_raw_meta(TAOS_RES* res, void** raw_meta, int32_t* raw_meta_len) { if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; *raw_meta = pMetaRspObj->metaRsp.metaRsp; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9eec1558b2..cca3a58588 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -306,7 +306,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE || pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE || pHead->msgType == TDMT_VND_DROP_TTL_TABLE); - // return + tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType); SMqMetaRsp metaRsp = {0}; metaRsp.reqOffset = pReq->currentOffset; metaRsp.rspOffset = fetchOffset; From 7fda5a861e58595fb0ad07f5a00987c7016a7dbb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 16:37:36 +0800 Subject: [PATCH 48/54] fix(query): fix to_ISO8601 function crash when processing constant illegal inputs TD-15211 --- source/libs/function/src/builtins.c | 2 +- source/libs/scalar/src/sclfunc.c | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index f6d2fdadae..f985cc324d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1268,7 +1268,7 @@ static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) { } static bool validateTimestampDigits(const SValueNode* pVal) { - if (IS_INTEGER_TYPE(pVal->node.resType.type)) { + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { return false; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index d335b04ea4..a5154fac39 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -856,10 +856,13 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); + bool tzPresent = (inputNum == 2) ? true : false; char* tz; int32_t tzLen; - tz = varDataVal(pInput[1].columnData->pData); - tzLen = varDataLen(pInput[1].columnData->pData); + if (tzPresent) { + tz = varDataVal(pInput[1].columnData->pData); + tzLen = varDataLen(pInput[1].columnData->pData); + } for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { From 58836d2ddf9e0b03f480759f10dc2cd6d60967eb Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 17:35:55 +0800 Subject: [PATCH 49/54] feat: refactor rpc quit --- source/libs/transport/src/transCli.c | 38 +++++++++++++++------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index bf03ea26ba..0dd00e0dc2 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -983,17 +983,18 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (pCtx->retryCount < pEpSet->numOfEps * 3) { pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + if (pThrd->quit == false) { + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + transPrintEpSet(pEpSet); + tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, pEpSet->numOfEps * 3); - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - transPrintEpSet(pEpSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, pEpSet->numOfEps * 3); - - transUnrefCliHandle(pConn); - return -1; + transUnrefCliHandle(pConn); + return -1; + } } } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { if (pResp->contLen == 0) { @@ -1010,15 +1011,16 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); } - if (pConn->status != ConnInPool) { - addConnToPool(pThrd->pool, pConn); + if (pThrd->quit == false) { + if (pConn->status != ConnInPool) { + addConnToPool(pThrd->pool, pConn); + } + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + return -1; } - - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - return -1; } } From 9c6057caa38c640887597bb690d20548e8761de2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 17:44:58 +0800 Subject: [PATCH 50/54] fix(query): fix syntax error. --- source/dnode/vnode/src/tq/tqExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 2d9cfbc0a1..da052ee3eb 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -75,7 +75,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp if (pDataBlock == NULL) break; ASSERT(pDataBlock->info.rows != 0); - ASSERT(pDataBlock->info.numOfCols != 0); + ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0); tqAddBlockDataToRsp(pDataBlock, pRsp); pRsp->blockNum++; From 9278de28b41c57a1796f617958f712b867ba615c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 23 Jun 2022 17:45:16 +0800 Subject: [PATCH 51/54] fix: improve udfd exist speed --- source/libs/function/src/udfd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index f8713e4b47..838071dbf1 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -939,8 +939,8 @@ int main(int argc, char *argv[]) { uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); udfdRun(); + removeListeningPipe(); - uv_thread_join(&mnodeConnectThread); udfdCloseClientRpc(); return 0; From c93d93e2d5ebffe2d56f4b57640c9f0d8e9c7fda Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jun 2022 18:23:51 +0800 Subject: [PATCH 52/54] enh: add stbs/tbs/dbs to monitor info --- include/libs/monitor/monitor.h | 3 +++ source/dnode/mnode/impl/src/mndMain.c | 3 +++ source/libs/monitor/src/monMain.c | 3 +++ source/libs/monitor/src/monMsg.c | 6 ++++++ 4 files changed, 15 insertions(+) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 39e8042b93..9e7aa4e27f 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -109,6 +109,9 @@ typedef struct { char version[MON_VER_LEN]; float master_uptime; // day int32_t monitor_interval; // sec + int32_t dbs_total; + int32_t stbs_total; + int64_t tbs_total; int32_t vgroups_total; int32_t vgroups_alive; int32_t vnodes_total; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index dede1c45e6..7b16adfb73 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -632,6 +632,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version)); pClusterInfo->monitor_interval = tsMonitorInterval; pClusterInfo->connections_total = mndGetNumOfConnections(pMnode); + pClusterInfo->dbs_total = sdbGetSize(pSdb, SDB_DB); + pClusterInfo->stbs_total = sdbGetSize(pSdb, SDB_STB); void *pIter = NULL; while (1) { @@ -681,6 +683,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr if (pIter == NULL) break; pClusterInfo->vgroups_total++; + pClusterInfo->tbs_total += pVgroup->numOfTables; SMonVgroupDesc desc = {0}; desc.vgroup_id = pVgroup->vgId; diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index bf857ad718..4d094e5931 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -201,6 +201,9 @@ static void monGenClusterJson(SMonInfo *pMonitor) { tjsonAddStringToObject(pJson, "version", pInfo->version); tjsonAddDoubleToObject(pJson, "master_uptime", pInfo->master_uptime); tjsonAddDoubleToObject(pJson, "monitor_interval", pInfo->monitor_interval); + tjsonAddDoubleToObject(pJson, "dbs_total", pInfo->dbs_total); + tjsonAddDoubleToObject(pJson, "tbs_total", pInfo->tbs_total); + tjsonAddDoubleToObject(pJson, "stbs_total", pInfo->stbs_total); tjsonAddDoubleToObject(pJson, "vgroups_total", pInfo->vgroups_total); tjsonAddDoubleToObject(pJson, "vgroups_alive", pInfo->vgroups_alive); tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total); diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c index a041b582a9..ced493d5b7 100644 --- a/source/libs/monitor/src/monMsg.c +++ b/source/libs/monitor/src/monMsg.c @@ -102,6 +102,9 @@ int32_t tEncodeSMonClusterInfo(SEncoder *encoder, const SMonClusterInfo *pInfo) if (tEncodeCStr(encoder, pInfo->version) < 0) return -1; if (tEncodeFloat(encoder, pInfo->master_uptime) < 0) return -1; if (tEncodeI32(encoder, pInfo->monitor_interval) < 0) return -1; + if (tEncodeI32(encoder, pInfo->dbs_total) < 0) return -1; + if (tEncodeI32(encoder, pInfo->stbs_total) < 0) return -1; + if (tEncodeI64(encoder, pInfo->tbs_total) < 0) return -1; if (tEncodeI32(encoder, pInfo->vgroups_total) < 0) return -1; if (tEncodeI32(encoder, pInfo->vgroups_alive) < 0) return -1; if (tEncodeI32(encoder, pInfo->vnodes_total) < 0) return -1; @@ -130,6 +133,9 @@ int32_t tDecodeSMonClusterInfo(SDecoder *decoder, SMonClusterInfo *pInfo) { if (tDecodeCStrTo(decoder, pInfo->version) < 0) return -1; if (tDecodeFloat(decoder, &pInfo->master_uptime) < 0) return -1; if (tDecodeI32(decoder, &pInfo->monitor_interval) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->dbs_total) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->stbs_total) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->tbs_total) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vgroups_total) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vgroups_alive) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vnodes_total) < 0) return -1; From 800098a79536aff581b5d16f67ef2ba964b9ab52 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 18:54:07 +0800 Subject: [PATCH 53/54] fix(query): fix syntax error on windows platform. --- source/libs/scalar/test/scalar/scalarTests.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index e49c11c390..07440e7435 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -86,7 +86,8 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s if (newBlock) { SSDataBlock *res = createDataBlock(); - SColumnInfoData idata = {.info = *colInfo}; + SColumnInfoData idata = {0}; + idata.info = *colInfo; colInfoDataEnsureCapacity(&idata, rows); blockDataAppendColInfo(res, &idata); @@ -263,9 +264,6 @@ void scltMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNod *pNode = (SNode *)onode; } - - - } TEST(constantTest, bigint_add_bigint) { From df1f4cc53faa22869e1ba473652f399d8972a2ee Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 23 Jun 2022 19:34:10 +0800 Subject: [PATCH 54/54] fix compilation error --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 690319a4cf..5f96b9e287 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -561,7 +561,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { // check for overflow if (IS_FLOAT_TYPE(type) && (isinf(pSumRes->dsum) || isnan(pSumRes->dsum))) { - numOfElems = 0; + numOfElem = 0; } _sum_over: