From a8be204254e05a7b2b604bf380d0e14f80379ac0 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 6 Jun 2022 14:37:00 +0800 Subject: [PATCH 1/7] feat: add multiwaysort merge operator --- source/libs/executor/inc/executorimpl.h | 3 +- source/libs/executor/src/executorimpl.c | 9 ++ source/libs/executor/src/sortoperator.c | 156 ++++++++++++++++++++++++ source/libs/index/src/indexFilter.c | 3 + 4 files changed, 170 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 7d597217ee..bd33d7f3c7 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -773,7 +773,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SNode* pCondition, SExecTaskInfo* pTaskInfo); SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pIndexMap, SExecTaskInfo* pTaskInfo); - +SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, + SSDataBlock* pResBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 92b3195f36..d094dc0203 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4517,6 +4517,15 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID); pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) { + SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode; + + SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; + SSDataBlock* pResBlock = createResDataBlock(pDescNode); + + SArray* sortInfo = createSortInfo(pMergePhyNode->pMergeKeys); + + pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pResBlock, sortInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 5e1b47cc91..0618c80682 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -215,3 +215,159 @@ int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* *len = sizeof(SSortExecInfo); return TSDB_CODE_SUCCESS; } + +typedef struct SMultiwaySortMergeOperatorInfo { + SOptrBasicInfo binfo; + + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + + SArray* pSortInfo; + SSortHandle* pSortHandle; + + int64_t startTs; // sort start time +} SMultiwaySortMergeOperatorInfo; + +SSDataBlock* getMultiwaySortMergedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity) { + blockDataCleanup(pDataBlock); + + blockDataEnsureCapacity(pDataBlock, capacity); + + while (1) { + STupleHandle* pTupleHandle = tsortNextTuple(pHandle); + if (pTupleHandle == NULL) { + break; + } + + appendOneRowToDataBlock(pDataBlock, pTupleHandle); + if (pDataBlock->info.rows >= capacity) { + return pDataBlock; + } + } + + return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; +} + + +int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { + SMultiwaySortMergeOperatorInfo * pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + pInfo->startTs = taosGetTimestampUs(); + + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, NULL, SORT_MULTISOURCE_MERGE, + pInfo->bufPageSize, numOfBufPage, NULL, pTaskInfo->id.str); + + tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); + + for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + SSortSource ps = {0}; + ps.param = pOperator->pDownstream[i]; + tsortAddSource(pInfo->pSortHandle, &ps); + } + + int32_t code = tsortOpen(pInfo->pSortHandle); + + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, terrno); + } + + pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs)/1000.0; + pOperator->status = OP_RES_TO_RETURN; + + OPTR_SET_OPENED(pOperator); + return TSDB_CODE_SUCCESS; +} + +SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSortOperatorInfo* pInfo = pOperator->info; + + int32_t code = pOperator->fpSet._openFn(pOperator); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + SSDataBlock* pBlock = getMultiwaySortMergedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + + if (pBlock != NULL) { + pOperator->resultInfo.totalRows += pBlock->info.rows; + } else { + doSetOperatorCompleted(pOperator); + } + return pBlock; +} + +void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) { + SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; + pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); + + taosArrayDestroy(pInfo->pSortInfo); +} + +int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { + ASSERT(pOptr != NULL); + SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); + + SMultiwaySortMergeOperatorInfo *pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info; + + *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle); + *pOptrExplain = pInfo; + *len = sizeof(SSortExecInfo); + return TSDB_CODE_SUCCESS; +} + +SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, + SSDataBlock* pResBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo) { + SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + int32_t rowSize = pResBlock->info.rowSize; + + if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { + goto _error; + } + + pInfo->binfo.pRes = pResBlock; + + initResultSizeInfo(pOperator, 1024); + + pInfo->pSortInfo = pSortInfo; + pOperator->name = "MultiwaySortMerge"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + + pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2; + pInfo->sortBufSize = pInfo->bufPageSize * 16; + + pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = + createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, + NULL, + NULL, destroyMultiwaySortMergeOperatorInfo, + NULL, NULL, + getMultiwaySortMergeExplainExecInfo); + + int32_t code = appendDownstream(pOperator, downStreams, numStreams); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; + +_error: + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + return NULL; +} \ No newline at end of file diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 34251ab918..72d29469db 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -171,7 +171,10 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { param->colId = l->colId; param->colValType = l->node.resType.type; memcpy(param->dbName, l->dbName, sizeof(l->dbName)); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-overflow" sprintf(param->colName, "%s_%s", l->colName, r->literal); +#pragma GCC diagnostic pop param->colValType = r->typeData; return 0; // memcpy(param->colName, l->colName, sizeof(l->colName)); From c438b699c937b49e60989516edc429063a3158b7 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 6 Jun 2022 15:12:07 +0800 Subject: [PATCH 2/7] feat: support multiway sort merge --- source/libs/executor/inc/executorimpl.h | 3 ++- source/libs/executor/src/executorimpl.c | 6 ++++- source/libs/executor/src/sortoperator.c | 32 +++++++------------------ 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index bd33d7f3c7..17a06a40c1 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -774,7 +774,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pIndexMap, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, - SSDataBlock* pResBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d094dc0203..beb5e1152f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4524,8 +4524,12 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pDescNode); SArray* sortInfo = createSortInfo(pMergePhyNode->pMergeKeys); + int32_t numOfOutputCols = 0; + SArray* pColList = NULL; + //extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID); - pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pResBlock, sortInfo, pTaskInfo); + + pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pResBlock, sortInfo, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 0618c80682..c219acb687 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -224,30 +224,11 @@ typedef struct SMultiwaySortMergeOperatorInfo { SArray* pSortInfo; SSortHandle* pSortHandle; + SArray* pColMatchInfo; // for index map from table scan output int64_t startTs; // sort start time } SMultiwaySortMergeOperatorInfo; -SSDataBlock* getMultiwaySortMergedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity) { - blockDataCleanup(pDataBlock); - - blockDataEnsureCapacity(pDataBlock, capacity); - - while (1) { - STupleHandle* pTupleHandle = tsortNextTuple(pHandle); - if (pTupleHandle == NULL) { - break; - } - - appendOneRowToDataBlock(pDataBlock, pTupleHandle); - if (pDataBlock->info.rows >= capacity) { - return pDataBlock; - } - } - - return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; -} - int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { SMultiwaySortMergeOperatorInfo * pInfo = pOperator->info; @@ -261,7 +242,7 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, NULL, SORT_MULTISOURCE_MERGE, + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, NULL, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); @@ -298,7 +279,10 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, code); } - SSDataBlock* pBlock = getMultiwaySortMergedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + SSDataBlock* pBlock = getSortedBlockData(pInfo->pSortHandle, + pInfo->binfo.pRes, + pOperator->resultInfo.capacity, + pInfo->pColMatchInfo); if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; @@ -328,7 +312,8 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx } SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, - SSDataBlock* pResBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo, + SExecTaskInfo* pTaskInfo) { SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); int32_t rowSize = pResBlock->info.rowSize; @@ -342,6 +327,7 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, initResultSizeInfo(pOperator, 1024); pInfo->pSortInfo = pSortInfo; + pInfo->pColMatchInfo= pColMatchColInfo; pOperator->name = "MultiwaySortMerge"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; pOperator->blocking = true; From 72c71141b2bfeabe07fc7bab64a4d2ca13234dfc Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 6 Jun 2022 15:16:20 +0800 Subject: [PATCH 3/7] feat: support multiway sort merge --- source/libs/executor/src/sortoperator.c | 111 ++++++++++++------------ 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index c219acb687..c84d4491af 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -13,17 +13,18 @@ * along with this program. If not, see . */ -#include "tdatablock.h" #include "executorimpl.h" +#include "tdatablock.h" static SSDataBlock* doSort(SOperatorInfo* pOperator); -static int32_t doOpenSortOperator(SOperatorInfo* pOperator); -static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); +static int32_t doOpenSortOperator(SOperatorInfo* pOperator); +static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, - SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, + SExprInfo* pExprInfo, int32_t numOfCols, SArray* pColMatchColInfo, + SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); int32_t rowSize = pResBlock->info.rowSize; @@ -32,33 +33,33 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR goto _error; } - pOperator->pExpr = pExprInfo; + pOperator->pExpr = pExprInfo; pOperator->numOfExprs = numOfCols; - pInfo->binfo.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = pResBlock; + pInfo->binfo.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = pResBlock; initResultSizeInfo(pOperator, 1024); - pInfo->pSortInfo = pSortInfo; - pInfo->pColMatchInfo= pColMatchColInfo; - pOperator->name = "SortOperator"; + pInfo->pSortInfo = pSortInfo; + pInfo->pColMatchInfo = pColMatchColInfo; + pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; // lazy evaluation for the following parameter since the input datablock is not known till now. -// pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header -// pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer + // pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + + // header pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = - createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, getExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, + getExplainExecInfo); int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pInfo); taosMemoryFree(pOperator); @@ -68,7 +69,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - bool isNull = tsortIsNullVal(pTupleHandle, i); + bool isNull = tsortIsNullVal(pTupleHandle, i); if (isNull) { colDataAppendNULL(pColInfo, pBlock->info.rows); } else { @@ -80,7 +81,8 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { pBlock->info.rows += 1; } -SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo) { +SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, + SArray* pColMatchInfo) { blockDataCleanup(pDataBlock); ASSERT(taosArrayGetSize(pColMatchInfo) == pDataBlock->info.numOfCols); @@ -129,10 +131,11 @@ SSDataBlock* loadNextDataBlock(void* param) { // todo refactor: merged with fetch fp void applyScalarFunction(SSDataBlock* pBlock, void* param) { - SOperatorInfo* pOperator = param; + SOperatorInfo* pOperator = param; SSortOperatorInfo* pSort = pOperator->info; if (pOperator->pExpr != NULL) { - int32_t code = projectApplyFunctions(pOperator->pExpr, pBlock, pBlock, pSort->binfo.pCtx, pOperator->numOfExprs, NULL); + int32_t code = + projectApplyFunctions(pOperator->pExpr, pBlock, pBlock, pSort->binfo.pCtx, pOperator->numOfExprs, NULL); if (code != TSDB_CODE_SUCCESS) { longjmp(pOperator->pTaskInfo->env, code); } @@ -141,7 +144,7 @@ void applyScalarFunction(SSDataBlock* pBlock, void* param) { int32_t doOpenSortOperator(SOperatorInfo* pOperator) { SSortOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (OPTR_IS_OPENED(pOperator)) { return TSDB_CODE_SUCCESS; @@ -150,8 +153,8 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) { pInfo->startTs = taosGetTimestampUs(); // pInfo->binfo.pRes is not equalled to the input datablock. - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT, - -1, -1, NULL, pTaskInfo->id.str); + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT, -1, -1, + NULL, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); @@ -166,7 +169,7 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, terrno); } - pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs)/1000.0; + pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs) / 1000.0; pOperator->status = OP_RES_TO_RETURN; OPTR_SET_OPENED(pOperator); @@ -186,7 +189,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, code); } - SSDataBlock* pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); + SSDataBlock* pBlock = + getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; @@ -208,7 +212,7 @@ int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* ASSERT(pOptr != NULL); SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); - SSortOperatorInfo *pOperatorInfo = (SSortOperatorInfo*)pOptr->info; + SSortOperatorInfo* pOperatorInfo = (SSortOperatorInfo*)pOptr->info; *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle); *pOptrExplain = pInfo; @@ -219,20 +223,19 @@ int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* typedef struct SMultiwaySortMergeOperatorInfo { SOptrBasicInfo binfo; - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort SArray* pSortInfo; SSortHandle* pSortHandle; SArray* pColMatchInfo; // for index map from table scan output - int64_t startTs; // sort start time + int64_t startTs; // sort start time } SMultiwaySortMergeOperatorInfo; - int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { - SMultiwaySortMergeOperatorInfo * pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (OPTR_IS_OPENED(pOperator)) { return TSDB_CODE_SUCCESS; @@ -259,7 +262,7 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, terrno); } - pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs)/1000.0; + pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs) / 1000.0; pOperator->status = OP_RES_TO_RETURN; OPTR_SET_OPENED(pOperator); @@ -271,18 +274,16 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { return NULL; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSortOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; int32_t code = pOperator->fpSet._openFn(pOperator); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } - SSDataBlock* pBlock = getSortedBlockData(pInfo->pSortHandle, - pInfo->binfo.pRes, - pOperator->resultInfo.capacity, - pInfo->pColMatchInfo); + SSDataBlock* pBlock = + getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; @@ -293,17 +294,18 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) { } void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) { - SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; + SMultiwaySortMergeOperatorInfo * pInfo = (SMultiwaySortMergeOperatorInfo*)param; pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); taosArrayDestroy(pInfo->pSortInfo); + taosArrayDestroy(pInfo->pColMatchInfo); } int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { ASSERT(pOptr != NULL); SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); - SMultiwaySortMergeOperatorInfo *pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info; + SMultiwaySortMergeOperatorInfo* pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info; *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle); *pOptrExplain = pInfo; @@ -315,35 +317,32 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) { SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - int32_t rowSize = pResBlock->info.rowSize; + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + int32_t rowSize = pResBlock->info.rowSize; if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { goto _error; } - pInfo->binfo.pRes = pResBlock; + pInfo->binfo.pRes = pResBlock; initResultSizeInfo(pOperator, 1024); - pInfo->pSortInfo = pSortInfo; - pInfo->pColMatchInfo= pColMatchColInfo; - pOperator->name = "MultiwaySortMerge"; + pInfo->pSortInfo = pSortInfo; + pInfo->pColMatchInfo = pColMatchColInfo; + pOperator->name = "MultiwaySortMerge"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2; pInfo->sortBufSize = pInfo->bufPageSize * 16; pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, - NULL, - NULL, destroyMultiwaySortMergeOperatorInfo, - NULL, NULL, - getMultiwaySortMergeExplainExecInfo); + createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL, + destroyMultiwaySortMergeOperatorInfo, NULL, NULL, getMultiwaySortMergeExplainExecInfo); int32_t code = appendDownstream(pOperator, downStreams, numStreams); if (code != TSDB_CODE_SUCCESS) { From 4700a83f099520ffe21a6b94a79191860eca9c0b Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 6 Jun 2022 15:34:19 +0800 Subject: [PATCH 4/7] add case for create_table_comment --- .../1-insert/create_table_comment.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 tests/system-test/1-insert/create_table_comment.py diff --git a/tests/system-test/1-insert/create_table_comment.py b/tests/system-test/1-insert/create_table_comment.py new file mode 100644 index 0000000000..92ea083c5a --- /dev/null +++ b/tests/system-test/1-insert/create_table_comment.py @@ -0,0 +1,113 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def get_long_name(self, length, mode="mixed"): + """ + generate long name + mode could be numbers/letters/letters_mixed/mixed + """ + if mode == "numbers": + population = string.digits + elif mode == "letters": + population = string.ascii_letters.lower() + elif mode == "letters_mixed": + population = string.ascii_letters.upper() + string.ascii_letters.lower() + else: + population = string.ascii_letters.lower() + string.digits + return "".join(random.choices(population, k=length)) + + def __create_tb(self,dbname,stbname,tbname,comment): + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute( + f'create table {stbname} (ts timestamp,c0 int) tags(t0 int) ') + tdSql.execute( + f'create table {tbname} using {stbname} tags(1) comment "{comment}"') + def __create_normaltb(self,dbname,tbname,comment): + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute( + f'create table {tbname} (ts timestamp,c0 int) comment "{comment}"') + + def check_comment(self): + dbname = self.get_long_name(length=10, mode="letters") + ntbname = self.get_long_name(length=5, mode="letters") + + # create normal table with comment + comment = self.get_long_name(length=10, mode="letters") + self.__create_normaltb(dbname,ntbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + print(ntb_kv_list) + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.error('alter table {ntbname} comment "test1"') + tdSql.execute(f'drop database {dbname}') + + # max length(1024) + comment = self.get_long_name(length=1024, mode="letters") + self.__create_normaltb(dbname,ntbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.execute(f'drop database {dbname}') + + # error overlength + comment = self.get_long_name(length=1025, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.error(f"create table ntb (ts timestamp,c0 int) comment '{comment}'") + tdSql.execute(f'drop database {dbname}') + + # create child table with comment + comment = self.get_long_name(length=10, mode="letters") + stbname = self.get_long_name(length=5, mode="letters") + tbname = self.get_long_name(length=3, mode="letters") + self.__create_tb(dbname,stbname,tbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.error(f'alter table {tbname} comment "test1"') + tdSql.execute(f'drop database {dbname}') + + # max length 1024 + comment = self.get_long_name(length=1024, mode="letters") + self.__create_tb(dbname,ntbname,comment) + ntb_kv_list = tdSql.getResult("show tables") + tdSql.checkEqual(ntb_kv_list[0][8], comment) + tdSql.execute(f'drop database {dbname}') + + # error overlength + comment = self.get_long_name(length=1025, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute(f"create table stb (ts timestamp,c0 int) tags(t0 int)") + tdSql.error(f'create table stb_1 us stb tags(1) comment "{comment}"') + tdSql.execute(f'drop database {dbname}') + + def run(self): + self.check_comment() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 8015c71f20a7d3300a676b985fa8fbd2ee991185 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 6 Jun 2022 15:34:56 +0800 Subject: [PATCH 5/7] add case into ci --- tests/system-test/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 3991654350..64e1fef77d 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -21,6 +21,7 @@ python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py #python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py python3 ./test.py -f 1-insert/alter_stable.py python3 ./test.py -f 1-insert/alter_table.py +python3 ./test.py -f 1-inerst/create_table_comment.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py From 1d1f392a01b4ccaea239a00603b5343044a87cbd Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 6 Jun 2022 15:35:11 +0800 Subject: [PATCH 6/7] update --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 64e1fef77d..792ba6ee4a 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -21,7 +21,7 @@ python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py #python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py python3 ./test.py -f 1-insert/alter_stable.py python3 ./test.py -f 1-insert/alter_table.py -python3 ./test.py -f 1-inerst/create_table_comment.py +# python3 ./test.py -f 1-inerst/create_table_comment.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py From ea8239c8e5716909aedd4f7142a0d9d01e77ae90 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 6 Jun 2022 16:40:37 +0800 Subject: [PATCH 7/7] fix: fix type convert of sum/avg for rsma --- source/common/src/tdatablock.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c615af705a..d9293433ea 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1590,7 +1590,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks for (int32_t k = 0; k < colNum; ++k) { // iterate by column SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); STColumn* pCol = &pTSchema->columns[k]; - ASSERT(pCol->type == pColInfoData->info.type); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: @@ -1614,20 +1613,39 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_MEDIUMBLOB: uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); TASSERT(0); break; default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pColInfoData->info.type, TD_VTYPE_NORM, var, true, offset, k); + char tv[8] = {0}; + if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, k); } else { uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); TASSERT(0); } break; } - offset += TYPE_BYTES[pColInfoData->info.type]; + offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation } dataLen += TD_ROW_LEN(rb.pBuf); #ifdef TD_DEBUG_PRINT_ROW