From a62a7cc41f709c829e7ebe8ef0bbe0f3807f1307 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 1 Aug 2022 21:05:44 +0800 Subject: [PATCH 01/65] enh: aggregate and window add field for merge data blocks --- include/libs/nodes/plannodes.h | 2 ++ source/libs/nodes/src/nodesCodeFuncs.c | 17 ++++++++-- source/libs/planner/src/planLogicCreater.c | 20 +++++++----- source/libs/planner/src/planPhysiCreater.c | 3 ++ source/libs/planner/src/planSpliter.c | 4 +++ source/libs/planner/test/planGroupByTest.cpp | 34 ++++++++++---------- 6 files changed, 53 insertions(+), 27 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 2f6bb603c1..6f25cc45fb 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -377,6 +377,7 @@ typedef struct SAggPhysiNode { SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function SNodeList* pGroupKeys; SNodeList* pAggFuncs; + bool mergeDataBlock; } SAggPhysiNode; typedef struct SDownstreamSourceNode { @@ -415,6 +416,7 @@ typedef struct SWinodwPhysiNode { int8_t igExpired; EOrder inputTsOrder; EOrder outputTsOrder; + bool mergeDataBlock; } SWinodwPhysiNode; typedef struct SIntervalPhysiNode { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 9d15b01acf..74659179a9 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1773,6 +1773,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { static const char* jkAggPhysiPlanExprs = "Exprs"; static const char* jkAggPhysiPlanGroupKeys = "GroupKeys"; static const char* jkAggPhysiPlanAggFuncs = "AggFuncs"; +static const char* jkAggPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) { const SAggPhysiNode* pNode = (const SAggPhysiNode*)pObj; @@ -1787,6 +1788,9 @@ static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkAggPhysiPlanAggFuncs, pNode->pAggFuncs); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanMergeDataBlock, pNode->mergeDataBlock); + } return code; } @@ -1804,6 +1808,9 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkAggPhysiPlanAggFuncs, &pNode->pAggFuncs); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkAggPhysiPlanMergeDataBlock, &pNode->mergeDataBlock); + } return code; } @@ -1936,8 +1943,9 @@ static const char* jkWindowPhysiPlanTsEnd = "TsEnd"; static const char* jkWindowPhysiPlanTriggerType = "TriggerType"; static const char* jkWindowPhysiPlanWatermark = "Watermark"; static const char* jkWindowPhysiPlanIgnoreExpired = "IgnoreExpired"; -static const char* jkWindowPhysiPlanInputTsOrder = "inputTsOrder"; +static const char* jkWindowPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkWindowPhysiPlanOutputTsOrder = "outputTsOrder"; +static const char* jkWindowPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; @@ -1970,6 +1978,9 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanOutputTsOrder, pNode->outputTsOrder); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkWindowPhysiPlanMergeDataBlock, pNode->mergeDataBlock); + } return code; } @@ -2005,6 +2016,9 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkWindowPhysiPlanOutputTsOrder, pNode->outputTsOrder, code); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkWindowPhysiPlanMergeDataBlock, &pNode->mergeDataBlock); + } return code; } @@ -2147,7 +2161,6 @@ static int32_t jsonToPhysiSessionWindowNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysiWindowNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkSessionWindowPhysiPlanGap, pNode->gap, code); - ; } return code; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b51624336b..a799b2feb6 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -468,6 +468,10 @@ static int32_t createGroupKeysFromPartKeys(SNodeList* pPartKeys, SNodeList** pOu return TSDB_CODE_SUCCESS; } +static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { + return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; +} + static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; @@ -480,7 +484,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->hasLastRow = pSelect->hasLastRowFunc; pAgg->hasTimeLineFunc = pSelect->hasTimeLineFunc; - pAgg->node.groupAction = GROUP_ACTION_SET; + pAgg->node.groupAction = getGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = pAgg->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; @@ -551,7 +555,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc; pIdfRowsFunc->isUniqueFunc = pSelect->hasUniqueFunc; pIdfRowsFunc->isTimeLineFunc = pSelect->hasTimeLineFunc; - pIdfRowsFunc->node.groupAction = GROUP_ACTION_KEEP; + pIdfRowsFunc->node.groupAction = getGroupAction(pCxt, pSelect); pIdfRowsFunc->node.requireDataOrder = pIdfRowsFunc->isTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; pIdfRowsFunc->node.resultDataOrder = pIdfRowsFunc->node.requireDataOrder; @@ -586,7 +590,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p return TSDB_CODE_OUT_OF_MEMORY; } - pInterpFunc->node.groupAction = GROUP_ACTION_KEEP; + pInterpFunc->node.groupAction = getGroupAction(pCxt, pSelect); pInterpFunc->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder; @@ -662,7 +666,7 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo } pWindow->winType = WINDOW_TYPE_STATE; - pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pWindow->pStateExpr = nodesCloneNode(pState->pExpr); @@ -685,7 +689,7 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW pWindow->winType = WINDOW_TYPE_SESSION; pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i; pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? SESSION_ALGO_STREAM_SINGLE : SESSION_ALGO_MERGE; - pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; @@ -714,7 +718,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; - pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); pWindow->node.requireDataOrder = pSelect->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_IN_BLOCK; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; @@ -762,7 +766,7 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return TSDB_CODE_OUT_OF_MEMORY; } - pFill->node.groupAction = GROUP_ACTION_KEEP; + pFill->node.groupAction = getGroupAction(pCxt, pSelect); pFill->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pFill->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pFill->inputTsOrder = ORDER_ASC; @@ -936,7 +940,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = GROUP_ACTION_SET; + pAgg->node.groupAction = GROUP_ACTION_CLEAR; pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 3771586b34..af3d4a9577 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -825,6 +825,8 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, return TSDB_CODE_OUT_OF_MEMORY; } + pAgg->mergeDataBlock = (GROUP_ACTION_KEEP == pAggLogicNode->node.groupAction ? false : true); + SNodeList* pPrecalcExprs = NULL; SNodeList* pGroupKeys = NULL; SNodeList* pAggFuncs = NULL; @@ -1091,6 +1093,7 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pWindow->igExpired = pWindowLogicNode->igExpired; pWindow->inputTsOrder = pWindowLogicNode->inputTsOrder; pWindow->outputTsOrder = pWindowLogicNode->outputTsOrder; + pWindow->mergeDataBlock = (GROUP_ACTION_KEEP == pWindowLogicNode->node.groupAction ? false : true); SNodeList* pPrecalcExprs = NULL; SNodeList* pFuncs = NULL; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 74c78b970d..6ddb6d90c0 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -393,6 +393,8 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic code = TSDB_CODE_OUT_OF_MEMORY; } + pPartWin->node.groupAction = GROUP_ACTION_KEEP; + if (TSDB_CODE_SUCCESS == code) { pMergeWindow->node.pTargets = pTargets; pMergeWindow->node.pConditions = pConditions; @@ -723,6 +725,8 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO code = TSDB_CODE_OUT_OF_MEMORY; } + pPartAgg->node.groupAction = GROUP_ACTION_KEEP; + if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { pPartAgg->pGroupKeys = pGroupKeys; code = createColumnByRewriteExprs(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets); diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index 657d8a6244..a553d3addc 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -32,6 +32,22 @@ TEST_F(PlanGroupByTest, basic) { run("SELECT c1 + c3, SUM(c4 * c5) FROM t1 WHERE CONCAT(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); run("SELECT SUM(CEIL(c1)) FROM t1 GROUP BY CEIL(c1)"); + + run("SELECT COUNT(*) FROM st1"); + + run("SELECT c1 FROM st1 GROUP BY c1"); + + run("SELECT COUNT(*) FROM st1 GROUP BY c1"); + + run("SELECT SUM(c1) FROM st1 GROUP BY c2 HAVING SUM(c1) IS NOT NULL"); +} + +TEST_F(PlanGroupByTest, withPartitionBy) { + useDb("root", "test"); + + run("SELECT LAST(ts), TBNAME FROM st1 PARTITION BY TBNAME"); + + run("SELECT COUNT(*) FROM st1 PARTITION BY c2 GROUP BY c1"); } TEST_F(PlanGroupByTest, withOrderBy) { @@ -43,14 +59,12 @@ TEST_F(PlanGroupByTest, withOrderBy) { // run("SELECT COUNT(*), SUM(c1) a FROM t1 ORDER BY a"); } -TEST_F(PlanGroupByTest, aggFunc) { +TEST_F(PlanGroupByTest, multiResFunc) { useDb("root", "test"); run("SELECT LAST(*), FIRST(*) FROM t1"); run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1"); - - run("SELECT SUM(10), COUNT(c1) FROM t1 GROUP BY c2"); } TEST_F(PlanGroupByTest, selectFunc) { @@ -67,17 +81,3 @@ TEST_F(PlanGroupByTest, selectFunc) { run("SELECT MAX(c1), c2 FROM t1 GROUP BY c3"); run("SELECT MAX(c1), t1.* FROM t1 GROUP BY c3"); } - -TEST_F(PlanGroupByTest, stable) { - useDb("root", "test"); - - run("SELECT COUNT(*) FROM st1"); - - run("SELECT c1 FROM st1 GROUP BY c1"); - - run("SELECT COUNT(*) FROM st1 GROUP BY c1"); - - run("SELECT COUNT(*) FROM st1 PARTITION BY c2 GROUP BY c1"); - - run("SELECT SUM(c1) FROM st1 GROUP BY c2 HAVING SUM(c1) IS NOT NULL"); -} From 781c494c2be7d2209eaa18ae99bbafeccf18af32 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 13:58:54 +0800 Subject: [PATCH 02/65] refactor: merge data belongs to different data group. --- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/executorimpl.c | 76 +++++++++---------- source/libs/executor/src/timewindowoperator.c | 1 + 3 files changed, 37 insertions(+), 44 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 577f9772be..e89459e555 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -519,6 +519,7 @@ typedef struct SBlockDistInfo { typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; SSDataBlock* pRes; + bool mergeResultBlock; } SOptrBasicInfo; typedef struct SIntervalAggOperatorInfo { @@ -593,7 +594,6 @@ typedef struct SAggOperatorInfo { uint64_t groupId; SGroupResInfo groupResInfo; SExprSupp scalarExprSup; - SNode *pCondition; } SAggOperatorInfo; @@ -891,7 +891,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, - int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); + int32_t numOfScalarExpr, bool mergeResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e52cbf40a9..a7b843d9c2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -136,9 +136,8 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, void operatorDummyCloseFn(void* param, int32_t numOfCols) {} -static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, - SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset, - SqlFunctionCtx* pCtx, int32_t numOfExprs); +static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); @@ -1501,19 +1500,24 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi return 0; } -int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, - SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, - int32_t numOfExprs) { - int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); - int32_t start = pGroupResInfo->index; +int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo) { + SExprInfo* pExprInfo = pSup->pExprInfo; + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; - for (int32_t i = start; i < numOfRows; i += 1) { + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + + for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); - doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset); + doUpdateNumOfRows(pRow, numOfExprs, rowEntryOffset); + + // no results, continue to check the next one if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; releaseBufPage(pBuf, page); @@ -1531,6 +1535,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); releaseBufPage(pBuf, page); break; } @@ -1540,7 +1545,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI for (int32_t j = 0; j < numOfExprs; ++j) { int32_t slotId = pExprInfo[j].base.resSchema.slotId; - pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowCellOffset); + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (pCtx[j].fpSet.finalize) { #ifdef BUF_PAGE_DEBUG qDebug("\npage_finalize %d", numOfExprs); @@ -1573,26 +1578,19 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI releaseBufPage(pBuf, page); pBlock->info.rows += pRow->numOfRows; - // if (pBlock->info.rows >= pBlock->info.capacity) { // output buffer is full - // break; - // } } qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.groupId); + blockDataUpdateTsWindow(pBlock, 0); return 0; } void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { - SExprInfo* pExprInfo = pOperator->exprSupp.pExprInfo; - int32_t numOfExprs = pOperator->exprSupp.numOfExprs; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int32_t* rowCellOffset = pOperator->exprSupp.rowEntryInfoOffset; - SSDataBlock* pBlock = pbInfo->pRes; - SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; + SSDataBlock* pBlock = pbInfo->pRes; // set output datablock version pBlock->info.version = pTaskInfo->version; @@ -1604,30 +1602,22 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG // clear the existed group id pBlock->info.groupId = 0; - doCopyToSDataBlock(pTaskInfo, pBlock, pExprInfo, pBuf, pGroupResInfo, rowCellOffset, pCtx, numOfExprs); -} - -static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, - int32_t* rowEntryInfoOffset) { - // update the number of result for each, only update the number of rows for the corresponding window result. - // if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - // return; - // } -#if 0 - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow* pResult = pResultRowInfo->pResult[i]; - - for (int32_t j = 0; j < numOfOutput; ++j) { - int32_t functionId = pCtx[j].functionId; - if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ) { - continue; + if (!pbInfo->mergeResultBlock) { + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo); + } else { + while(hasRemainResults(pGroupResInfo)) { + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo); + if (pBlock->info.rows >= pOperator->resultInfo.threshold) { + break; } - SResultRowEntryInfo* pCell = getResultEntryInfo(pResult, j, rowEntryInfoOffset); - pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); + // clearing group id to continue to merge data that belong to different groups + pBlock->info.groupId = 0; } + + // clear the group id info in SSDataBlock, since the client does not need it + pBlock->info.groupId = 0; } -#endif } static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, int8_t compressed) { @@ -2979,6 +2969,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { break; } } + size_t rows = blockDataGetNumOfRows(pInfo->pRes); pOperator->resultInfo.totalRows += rows; @@ -3483,7 +3474,7 @@ void cleanupExprSupp(SExprSupp* pSupp) { SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, - int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo) { + int32_t numOfScalarExpr, bool mergeResult, SExecTaskInfo* pTaskInfo) { SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3505,6 +3496,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } + pInfo->binfo.mergeResultBlock = mergeResult; pInfo->groupId = UINT64_MAX; pInfo->pCondition = pCondition; pOperator->name = "TableAggregate"; @@ -4098,7 +4090,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pScalarExprInfo, numOfScalarExpr, pTaskInfo); } else { pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, - pScalarExprInfo, numOfScalarExpr, pTaskInfo); + pScalarExprInfo, numOfScalarExpr, pAggNode->mergeDataBlock, pTaskInfo); } } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 802e1f2306..c447d6b43b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1798,6 +1798,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->twAggSup = *pTwAggSupp; pInfo->ignoreExpiredData = pPhyNode->window.igExpired; pInfo->pCondition = pPhyNode->window.node.pConditions; + pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock; if (pPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; From 770cc74e7f220b2465b3c0402f2c2bd9844c51c6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 14:19:15 +0800 Subject: [PATCH 03/65] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8821dbd5a1..43210ced2a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -464,6 +464,11 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pBlock->info = binfo; ASSERT(binfo.uid != 0); + uint64_t* groupId = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); + if (groupId) { + pBlock->info.groupId = *groupId; + } + uint32_t status = 0; int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); @@ -476,11 +481,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - uint64_t* groupId = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); - if (groupId) { - pBlock->info.groupId = *groupId; - } - pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows; pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; From be51233c241d906c9525d38e95c1943fb7a77413 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 14:26:54 +0800 Subject: [PATCH 04/65] refactor: add dynamic prune for last partial. --- source/libs/function/src/builtins.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6383179fee..23e26f7716 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2343,6 +2343,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_LAST_PARTIAL, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLastPartial, + .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastFunction, From cbed4713afccd050c7d4189dd460b9517eb993d9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 14:34:50 +0800 Subject: [PATCH 05/65] refactor: do some internal refactor. --- source/libs/function/src/functionMgt.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 339ff3808b..72ba81c3a7 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -116,6 +116,11 @@ EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, STimeWindow* return TSDB_CODE_FAILED; } + const char* name = funcMgtBuiltins[funcId].name; + if ((strcmp(name, "_group_key") == 0) || (strcmp(name, "_select_value") == 0)) { + return FUNC_DATA_REQUIRED_NOT_LOAD; + } + if (funcMgtBuiltins[funcId].dynDataRequiredFunc == NULL) { return FUNC_DATA_REQUIRED_DATA_LOAD; } else { From 258d50306fa5295fa21bb9434bdcdfcd7f8d0e71 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 15:43:45 +0800 Subject: [PATCH 06/65] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/cachescanoperator.c | 2 +- source/libs/executor/src/executorimpl.c | 5 +- source/libs/executor/src/timewindowoperator.c | 124 +++++++++--------- 4 files changed, 63 insertions(+), 72 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index e89459e555..d24156bc60 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -554,8 +554,6 @@ typedef struct SMergeAlignedIntervalAggOperatorInfo { uint64_t groupId; // current groupId int64_t curTs; // current ts SSDataBlock* prefetchedBlock; - bool inputBlocksFinished; - SNode* pCondition; } SMergeAlignedIntervalAggOperatorInfo; @@ -906,7 +904,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - SExecTaskInfo* pTaskInfo); + bool mergeResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 56a5e253d8..94e4384b30 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -50,7 +50,7 @@ SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pScanNode, SRead STableListInfo* pTableList = &pTaskInfo->tableqinfoList; - initResultSizeInfo(&pOperator->resultInfo, 1024); + initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); pInfo->pUidList = taosArrayInit(4, sizeof(int64_t)); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a7b843d9c2..794e751dd2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3481,10 +3481,9 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - int32_t numOfRows = 1024; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, numOfRows); + initResultSizeInfo(&pOperator->resultInfo, 4096); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -4147,7 +4146,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; - pOptr = createMergeIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pTaskInfo); + pOptr = createMergeIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pIntervalPhyNode->window.mergeDataBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL == type) { int32_t children = 0; pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c447d6b43b..812428920a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4622,20 +4622,25 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR int32_t numOfOutput = pSup->numOfExprs; int64_t* tsCols = extractTsCol(pBlock, iaInfo); uint64_t tableGroupId = pBlock->info.groupId; - TSKEY currTs = getStartTsKey(&pBlock->info.window, tsCols); SResultRow* pResult = NULL; + TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); + // there is an result exists if (miaInfo->curTs != INT64_MIN) { ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); - if (currTs != miaInfo->curTs) { + + if (ts != miaInfo->curTs) { outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs); - miaInfo->curTs = INT64_MIN; + miaInfo->curTs = ts; } + } else { + miaInfo->curTs = ts; + ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); } STimeWindow win = {0}; - win.skey = currTs; + win.skey = miaInfo->curTs; win.ekey = taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1; @@ -4646,12 +4651,11 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - miaInfo->curTs = win.skey; int32_t currPos = startPos; STimeWindow currWin = win; while (++currPos < pBlock->info.rows) { - if (tsCols[currPos] == currTs) { + if (tsCols[currPos] == miaInfo->curTs) { continue; } @@ -4659,11 +4663,10 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder); - outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs); - miaInfo->curTs = INT64_MIN; + outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs); + miaInfo->curTs = tsCols[currPos]; - currTs = tsCols[currPos]; - currWin.skey = currTs; + currWin.skey = miaInfo->curTs; currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1; @@ -4680,14 +4683,6 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder); - - if (currPos >= pBlock->info.rows) { - // we need to see next block if exists - } else { - ASSERT(0); - outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs); - miaInfo->curTs = INT64_MIN; - } } static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { @@ -4703,59 +4698,56 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SSDataBlock* pRes = iaInfo->binfo.pRes; blockDataCleanup(pRes); - blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); - if (!miaInfo->inputBlocksFinished) { - SOperatorInfo* downstream = pOperator->pDownstream[0]; - int32_t scanFlag = MAIN_SCAN; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + int32_t scanFlag = MAIN_SCAN; - while (1) { - SSDataBlock* pBlock = NULL; - if (miaInfo->prefetchedBlock == NULL) { - pBlock = downstream->fpSet.getNextFn(downstream); - } else { - pBlock = miaInfo->prefetchedBlock; - miaInfo->groupId = pBlock->info.groupId; - miaInfo->prefetchedBlock = NULL; - } + while (1) { + SSDataBlock* pBlock = NULL; + if (miaInfo->prefetchedBlock == NULL) { + pBlock = downstream->fpSet.getNextFn(downstream); + } else { + pBlock = miaInfo->prefetchedBlock; + miaInfo->prefetchedBlock = NULL; - if (pBlock == NULL) { - // close last unfinalized time window - if (miaInfo->curTs != INT64_MIN) { - ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); - outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); - miaInfo->curTs = INT64_MIN; - } - - doSetOperatorCompleted(pOperator); - miaInfo->inputBlocksFinished = true; - break; - } - - if (!miaInfo->hasGroupId) { - miaInfo->hasGroupId = true; - miaInfo->groupId = pBlock->info.groupId; - } else if (miaInfo->groupId != pBlock->info.groupId) { - // if there are unclosed time window, close it firstly. - ASSERT(miaInfo->curTs != INT64_MIN); - outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); - miaInfo->prefetchedBlock = pBlock; - miaInfo->curTs = INT64_MIN; - break; - } - - getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag); - setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true); - doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); - doFilter(miaInfo->pCondition, pRes, NULL); - if (pRes->info.rows >= pOperator->resultInfo.capacity) { - break; - } + miaInfo->groupId = pBlock->info.groupId; } - pRes->info.groupId = miaInfo->groupId; + if (pBlock == NULL) { + // close last unfinalized time window + if (miaInfo->curTs != INT64_MIN) { + ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); + outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); + miaInfo->curTs = INT64_MIN; + } + + doSetOperatorCompleted(pOperator); + break; + } + + if (!miaInfo->hasGroupId) { + miaInfo->hasGroupId = true; + miaInfo->groupId = pBlock->info.groupId; + } else if (miaInfo->groupId != pBlock->info.groupId) { + // if there are unclosed time window, close it firstly. + ASSERT(miaInfo->curTs != INT64_MIN); + outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); + miaInfo->prefetchedBlock = pBlock; + miaInfo->curTs = INT64_MIN; + break; + } + + getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag); + setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true); + doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); + + doFilter(miaInfo->pCondition, pRes, NULL); + if (pRes->info.rows >= pOperator->resultInfo.capacity) { + break; + } } + pRes->info.groupId = miaInfo->groupId; miaInfo->hasGroupId = false; size_t rows = pRes->info.rows; @@ -4809,6 +4801,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, } initResultRowInfo(&iaInfo->binfo.resultRowInfo); + blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity); pOperator->name = "TimeMergeAlignedIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL; @@ -5076,7 +5069,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - SExecTaskInfo* pTaskInfo) { + bool mergeBlock, SExecTaskInfo* pTaskInfo) { SMergeIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (miaInfo == NULL || pOperator == NULL) { @@ -5090,6 +5083,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI iaInfo->inputOrder = TSDB_ORDER_ASC; iaInfo->interval = *pInterval; iaInfo->execModel = pTaskInfo->execModel; + iaInfo->binfo.mergeResultBlock = mergeBlock; iaInfo->primaryTsIndex = primaryTsSlotId; From 70284f5b0202317b27c05f961ee476f2ba406d8a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 16:52:36 +0800 Subject: [PATCH 07/65] enh(query): support multi-groups result merge into one datablock. --- source/libs/executor/src/timewindowoperator.c | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 812428920a..5dcfb400d9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4685,20 +4685,16 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder); } -static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { + +static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperator->info; SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } SExprSupp* pSup = &pOperator->exprSupp; SSDataBlock* pRes = iaInfo->binfo.pRes; - blockDataCleanup(pRes); - SOperatorInfo* downstream = pOperator->pDownstream[0]; int32_t scanFlag = MAIN_SCAN; @@ -4749,6 +4745,35 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { pRes->info.groupId = miaInfo->groupId; miaInfo->hasGroupId = false; +} + +static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info; + SIntervalAggOperatorInfo* iaInfo = pMiaInfo->intervalAggOperatorInfo; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSDataBlock* pRes = iaInfo->binfo.pRes; + blockDataCleanup(pRes); + + if (iaInfo->binfo.mergeResultBlock) { + while(1) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pRes->info.rows >= pOperator->resultInfo.threshold) { + break; + } + + doMergeAlignedIntervalAgg(pOperator); + } + } else { + doMergeAlignedIntervalAgg(pOperator); + } size_t rows = pRes->info.rows; pOperator->resultInfo.totalRows += rows; @@ -4812,7 +4837,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->exprSupp.numOfExprs = numOfCols; pOperator->info = miaInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeAlignedIntervalAgg, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL, destroyMergeAlignedIntervalOperatorInfo, NULL, NULL, NULL); code = appendDownstream(pOperator, &downstream, 1); From f98945e111e28061ee3c22d3d0f497262ffe9ff9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 2 Aug 2022 18:57:55 +0800 Subject: [PATCH 08/65] change log level --- source/libs/transport/src/transCli.c | 4 ++-- source/libs/transport/src/transSvr.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 431e479123..7ab3102d1c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -381,7 +381,7 @@ void cliHandleResp(SCliConn* conn) { STraceId* trace = &transMsg.info.traceId; - tGTrace("%s conn %p %s received from %s, local info:%s, len:%d, code str:%s", CONN_GET_INST_LABEL(conn), conn, + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, code str:%s", CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(pHead->msgType), conn->dst, conn->src, transMsg.contLen, tstrerror(transMsg.code)); if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { @@ -757,7 +757,7 @@ void cliSend(SCliConn* pConn) { uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); STraceId* trace = &pMsg->info.traceId; - tGTrace("%s conn %p %s is sent to %s, local info %s, len:%d", CONN_GET_INST_LABEL(pConn), pConn, + tGDebug("%s conn %p %s is sent to %s, local info %s, len:%d", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, pMsg->contLen); if (pHead->persist == 1) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 8b27d95e52..ea9d7bb910 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -253,10 +253,10 @@ static void uvHandleReq(SSvrConn* pConn) { if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); - tGTrace("%s conn %p %s received from %s, local info:%s, len:%d", transLabel(pTransInst), pConn, + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, transMsg.contLen); } else { - tGTrace("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d", transLabel(pTransInst), pConn, + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, transMsg.contLen, pHead->noResp, transMsg.code); } @@ -416,7 +416,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { STrans* pTransInst = pConn->pTransInst; STraceId* trace = &pMsg->info.traceId; - tGTrace("%s conn %p %s is sent to %s, local info:%s, len:%d", transLabel(pTransInst), pConn, + tGDebug("%s conn %p %s is sent to %s, local info:%s, len:%d", transLabel(pTransInst), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, pMsg->contLen); pHead->msgLen = htonl(len); From 77ed2bc1e13056f47c1db7f64d800eec68aa6abd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 21:25:55 +0800 Subject: [PATCH 09/65] enh(query): add merge attribute in explain. --- source/libs/command/inc/commandInt.h | 3 ++- source/libs/command/src/explain.c | 18 +++++++++++++++--- source/libs/executor/inc/executorimpl.h | 6 +++--- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/timewindowoperator.c | 5 +++-- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 2ae4666ff6..c53d464eab 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -49,6 +49,7 @@ extern "C" { #define EXPLAIN_PARITION_FORMAT "Partition on Column %s" #define EXPLAIN_ORDER_FORMAT "Order: %s" #define EXPLAIN_FILTER_FORMAT "Filter: " +#define EXPLAIN_MERGEBLOCKS_FORMAT "Merge ResBlocks: %s" #define EXPLAIN_FILL_VALUE_FORMAT "Fill Values: " #define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " #define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" @@ -56,7 +57,7 @@ extern "C" { #define EXPLAIN_TIME_WINDOWS_FORMAT "Time Window: interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c" #define EXPLAIN_WINDOW_FORMAT "Window: gap=%" PRId64 #define EXPLAIN_RATIO_TIME_FORMAT "Ratio: %f" -#define EXPLAIN_MERGE_FORMAT "Merge" +#define EXPLAIN_MERGE_FORMAT "SortMerge" #define EXPLAIN_MERGE_KEYS_FORMAT "Merge Key: " #define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms" diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index c080b666cc..2836bd7acd 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -702,7 +702,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pAggNode->mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } @@ -901,7 +905,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pIntNode->window.mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } @@ -945,7 +953,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pIntNode->window.mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d24156bc60..6383ba06b3 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -908,10 +908,10 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - SNode* pCondition, SExecTaskInfo* pTaskInfo); + SNode* pCondition, bool mergeResultBlocks, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); +SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, int32_t numOfChild); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 794e751dd2..112b6ebc99 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4131,7 +4131,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, - pPhyNode->pConditions, pTaskInfo); + pPhyNode->pConditions, pIntervalPhyNode->window.mergeDataBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) { SMergeIntervalPhysiNode* pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5dcfb400d9..aaa7a1ea8d 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4762,7 +4762,7 @@ static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) { if (iaInfo->binfo.mergeResultBlock) { while(1) { if (pOperator->status == OP_EXEC_DONE) { - return NULL; + break; } if (pRes->info.rows >= pOperator->resultInfo.threshold) { @@ -4782,7 +4782,7 @@ static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, - int32_t primaryTsSlotId, SNode* pCondition, + int32_t primaryTsSlotId, SNode* pCondition, bool mergeResultBlock, SExecTaskInfo* pTaskInfo) { SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -4806,6 +4806,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, iaInfo->interval = *pInterval; iaInfo->execModel = pTaskInfo->execModel; iaInfo->primaryTsIndex = primaryTsSlotId; + iaInfo->binfo.mergeResultBlock = mergeResultBlock; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(&pOperator->resultInfo, 4096); From 64db9afeb7c49d063d8fb6801d356ad2428212c6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 10:52:06 +0800 Subject: [PATCH 10/65] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 84 +++++++++++++++----------- 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c003f5a63f..6d1e3a4e07 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1205,6 +1205,26 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* return code; } +static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, int64_t key, SFileBlockDumpInfo* pDumpInfo) { + + // opt version + // 1. it is not a border point + // 2. the direct next point is not an duplicated timestamp + if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && pReader->order == TSDB_ORDER_ASC) || + (pDumpInfo->rowIndex > 0 && pReader->order == TSDB_ORDER_DESC)) { + int32_t step = pReader->order == TSDB_ORDER_ASC? 1:-1; + + int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; + if (nextKey != key) { // merge is not needed + doAppendRowFromBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + pDumpInfo->rowIndex += step; + return true; + } + } + + return false; +} + static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key) { SRowMerger merge = {0}; @@ -1219,10 +1239,14 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* // ascending order traverse if (ASCENDING_TRAVERSE(pReader->order)) { if (key < k.ts) { - tRowMergerInit(&merge, &fRow, pReader->pSchema); - - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); + // imem & mem are all empty, only file exist + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + } } else if (k.ts < key) { // k.ts < key doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); } else { // k.ts == key, ascending order: file block ----> imem rows -----> mem rows @@ -1238,10 +1262,13 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (key < k.ts) { doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); } else if (k.ts < key) { - tRowMergerInit(&merge, &fRow, pReader->pSchema); - - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + } } else { // descending order: mem rows -----> imem rows ------> file block updateSchema(pRow, pBlockScanInfo->uid, pReader); @@ -1438,34 +1465,23 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } // imem & mem are all empty, only file exist + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - // opt version - // 1. it is not a border point - // 2. the direct next point is not an duplicated timestamp - if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && pReader->order == TSDB_ORDER_ASC) || - (pDumpInfo->rowIndex > 0 && pReader->order == TSDB_ORDER_DESC)) { - int32_t step = pReader->order == TSDB_ORDER_ASC? 1:-1; - int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; - if (nextKey != key) { // merge is not needed - doAppendRowFromBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); - pDumpInfo->rowIndex += step; - return TSDB_CODE_SUCCESS; - } + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; } - - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - - STSRow* pTSRow = NULL; - SRowMerger merge = {0}; - - tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); - - taosMemoryFree(pTSRow); - tRowMergerClear(&merge); - return TSDB_CODE_SUCCESS; } } From bc600a4942e5a82cad3d2749f613838f9315b33a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 14:08:03 +0800 Subject: [PATCH 11/65] refactor: do some internal refactor optimize the building block performance. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 76 +++++++++++++++++++------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6d1e3a4e07..62d7a72b8f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -152,7 +152,7 @@ static void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); static void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, - STsdbReader* pReader); + STsdbReader* pReader, bool* freeTSRow); static void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow); static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData, @@ -1235,6 +1235,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* TSDBKEY k = TSDBROW_KEY(pRow); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SArray* pDelList = pBlockScanInfo->delSkyline; + bool freeTSRow = false; // ascending order traverse if (ASCENDING_TRAVERSE(pReader->order)) { @@ -1248,7 +1249,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerGetRow(&merge, &pTSRow); } } else if (k.ts < key) { // k.ts < key - doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader, &freeTSRow); } else { // k.ts == key, ascending order: file block ----> imem rows -----> mem rows tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); @@ -1260,7 +1261,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } else { // descending order scan if (key < k.ts) { - doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader, &freeTSRow); } else if (k.ts < key) { if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { return TSDB_CODE_SUCCESS; @@ -1302,12 +1303,12 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* ASSERT(pRow != NULL && piRow != NULL); int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + bool freeTSRow = false; uint64_t uid = pBlockScanInfo->uid; TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); - if (ASCENDING_TRAVERSE(pReader->order)) { // [1&2] key <= [k.ts && ik.ts] if (key <= k.ts && key <= ik.ts) { @@ -1335,16 +1336,22 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [3] ik.ts < key <= k.ts // [4] ik.ts < k.ts <= key if (ik.ts < k.ts) { - doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader); + doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } return TSDB_CODE_SUCCESS; } // [5] k.ts < key <= ik.ts // [6] k.ts < ik.ts <= key if (k.ts < ik.ts) { - doMergeMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader); + doMergeMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader, &freeTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } return TSDB_CODE_SUCCESS; } @@ -1354,6 +1361,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } } @@ -1385,8 +1393,11 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [3] ik.ts > k.ts >= Key // [4] ik.ts > key >= k.ts if (ik.ts > key) { - doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader); + doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } return TSDB_CODE_SUCCESS; } @@ -1399,18 +1410,21 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } //[7] key = ik.ts > k.ts if (key == ik.ts) { - doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader); + doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); tRowMerge(&merge, &fRow); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + + taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } } @@ -2280,11 +2294,29 @@ void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader) { } void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, - STsdbReader* pReader) { - SRowMerger merge = {0}; + STsdbReader* pReader, bool* freeTSRow) { + { // if the timestamp of the next valid row has a different ts, return current row directly + TSDBROW current = *pRow; + pIter->hasVal = tsdbTbDataIterNext(pIter->iter); - TSDBKEY k = TSDBROW_KEY(pRow); - // updateSchema(pRow, uid, pReader); + if (!pIter->hasVal) { + *pTSRow = current.pTSRow; + *freeTSRow = false; + return; + } else { // has next point in mem/imem + TSDBROW* pNextRow = getValidRow(pIter, pDelList, pReader); + if (TSDBROW_KEY(¤t).ts != TSDBROW_KEY(pNextRow).ts) { + *pTSRow = current.pTSRow; + *freeTSRow = false; + return; + } + } + } + + SRowMerger merge = {0}; + TSDBKEY k = TSDBROW_KEY(pRow); + + // get the correct schema for data in memory int32_t sversion = TSDBROW_SVERSION(pRow); STSchema* pTSchema = NULL; if (sversion != pReader->pSchema->version) { @@ -2298,6 +2330,8 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe tRowMergerGetRow(&merge, pTSRow); tRowMergerClear(&merge); + *freeTSRow = true; + if (sversion != pReader->pSchema->version) { taosMemoryFree(pTSchema); } @@ -2332,7 +2366,7 @@ void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlo } int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow, - int64_t endKey) { + int64_t endKey, bool* freeTSRow) { TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); SArray* pDelList = pBlockScanInfo->delSkyline; @@ -2358,23 +2392,24 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR TSDBKEY ik = TSDBROW_KEY(piRow); if (ik.ts < k.ts) { // ik.ts < k.ts - doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader); + doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); } else if (k.ts < ik.ts) { - doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); } else { // ik.ts == k.ts doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, pTSRow); + *freeTSRow = true; } return TSDB_CODE_SUCCESS; } if (pBlockScanInfo->iter.hasVal && pRow != NULL) { - doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); return TSDB_CODE_SUCCESS; } if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { - doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader); + doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); return TSDB_CODE_SUCCESS; } @@ -2472,13 +2507,16 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e do { STSRow* pTSRow = NULL; - tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow, endKey); + bool freeTSRow = false; + tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow, endKey, &freeTSRow); if (pTSRow == NULL) { break; } doAppendRowFromTSRow(pBlock, pReader, pTSRow); - taosMemoryFree(pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } // no data in buffer, return immediately if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) { From 074500805c1745035d50217758115067360db5ae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 15:34:42 +0800 Subject: [PATCH 12/65] refactor: set filter function for last_row function. --- source/libs/executor/src/scanoperator.c | 5 ++--- source/libs/function/src/builtins.c | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 43210ced2a..b0f05e4705 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -521,7 +521,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; - qDebug("%s start to repeat ascending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); + qDebug("%s start to repeat ascending order scan data SELECT last_row(*),hostname from cpu group by hostname;blocks due to query func required", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); @@ -533,10 +533,9 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(pTableScanInfo, pOperator->exprSupp.pCtx, 0); tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); + qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); } - qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); - while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 23e26f7716..444cd2c5d0 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2271,6 +2271,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_LAST_ROW, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLast, + .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastRowFunction, From a3166507a54baf86a31d45f91935deecc9911229 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 15:58:28 +0800 Subject: [PATCH 13/65] enh: don't read bitmap when all col is normal --- include/common/trow.h | 8 ++-- source/client/src/tmq.c | 2 + source/common/src/tdatablock.c | 1 + source/common/src/trow.c | 56 +++++++++++++++---------- source/dnode/vnode/src/tq/tqSink.c | 1 + source/libs/executor/src/dataInserter.c | 1 + source/libs/parser/src/parInsert.c | 7 +++- source/libs/parser/src/parInsertData.c | 1 + 8 files changed, 52 insertions(+), 25 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 807a4c0f0a..93004b2cbd 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -38,7 +38,8 @@ typedef struct { uint16_t type : 2; uint16_t del : 1; uint16_t endian : 1; - uint16_t reserve : 12; + uint16_t normal : 1; // all norm + uint16_t reserve : 11; uint16_t sver; }; }; @@ -149,6 +150,8 @@ typedef struct { void *pBitmap; void *pOffset; int32_t extendedRowSize; + int16_t nNone; + int16_t nNull; } SRowBuilder; #define TD_ROW_HEAD_LEN (sizeof(STSRow)) @@ -287,9 +290,8 @@ int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen); int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, int32_t allNullLen, int32_t boundNullLen); int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); +int32_t tdSRowEnd(SRowBuilder *pBuilder); int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf); -int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, - int32_t nBoundCols, int32_t flen); void tdSRowReset(SRowBuilder *pBuilder); int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index bdd8c75f26..46349a3327 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -3129,6 +3129,7 @@ int taos_write_raw_block(TAOS *taos, int rows, char *pData, const char* tbname){ offset += TYPE_BYTES[pColumn->type]; } + tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); rowData = POINTER_SHIFT(rowData, rowLen); dataLen += rowLen; @@ -3360,6 +3361,7 @@ static int32_t tmqWriteRaw(TAOS *taos, void* data, int32_t dataLen){ } offset += TYPE_BYTES[pColumn->type]; } + tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); rowData = POINTER_SHIFT(rowData, rowLen); dataLen += rowLen; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 613fdd6b77..a16364b4a9 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1906,6 +1906,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks } offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation } + tdSRowEnd(&rb); dataLen += TD_ROW_LEN(rb.pBuf); #ifdef TD_DEBUG_PRINT_ROW tdSRowPrint(rb.pBuf, pTSchema, __func__); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 754e142437..6558af8aad 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -666,6 +666,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { ++iColVal; } + tdSRowEnd(&rb); taosMemoryFreeClear(varBuf); @@ -880,26 +881,20 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, int16_t colIdx) { -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { - output->valType = TD_VTYPE_NONE; - return terrno; - } - if (tdValTypeIsNorm(output->valType)) { + if (pRow->normal) { if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); } + return TSDB_CODE_SUCCESS; } -#else - if (IS_VAR_DATA_TYPE(colType)) { - output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + + if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { + output->valType = TD_VTYPE_NONE; + return terrno; } - output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif + return TSDB_CODE_SUCCESS; } @@ -925,6 +920,21 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp return TSDB_CODE_SUCCESS; } // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. + + switch (valType) { + case TD_VTYPE_NORM: + break; + case TD_VTYPE_NULL: + ++pBuilder->nNull; + break; + case TD_VTYPE_NONE: + ++pBuilder->nNone; + break; + default: + ASSERT(0); + break; + } + if (TD_IS_TP_ROW(pRow)) { tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset); } else { @@ -1084,6 +1094,14 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou return TSDB_CODE_SUCCESS; } +int32_t tdSRowEnd(SRowBuilder *pBuilder) { + STSRow *pRow = (STSRow *)pBuilder->pBuf; + if (pBuilder->nNone || pBuilder->nNull) { + pRow->normal = 1; + } + return TSDB_CODE_SUCCESS; +} + int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { @@ -1092,6 +1110,10 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { return terrno; } + if (pBuilder->nNone) pBuilder->nNone = 0; + if (pBuilder->nNull) pBuilder->nNull = 0; + + pBuilder->nNull = 0; TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); @@ -1159,14 +1181,6 @@ int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { return TSDB_CODE_SUCCESS; } -int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, - int32_t nBoundCols, int32_t flen) { - if (tdSRowSetExtendedInfo(pBuilder, allNullLen, boundNullLen, nCols, nBoundCols, flen) < 0) { - return terrno; - } - return tdSRowResetBuf(pBuilder, pBuf); -} - void tdSRowReset(SRowBuilder *pBuilder) { pBuilder->rowType = TD_ROW_TP; pBuilder->pBuf = NULL; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 850394b15d..fc6f2b0fa8 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -157,6 +157,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); } } + tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); rowData = POINTER_SHIFT(rowData, rowLen); dataLen += rowLen; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 1c08fafaa3..5e43e52afa 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -213,6 +213,7 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); } } + tdSRowEnd(&rb); if (ignoreRow) { continue; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index fbc5e1b4b0..d1cf82ff1f 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1206,6 +1206,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, } // *len = pBuilder->extendedRowSize; + tdSRowEnd(pBuilder); return TSDB_CODE_SUCCESS; } @@ -1973,6 +1974,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in } } } + tdSRowEnd(pBuilder); #ifdef TD_DEBUG_PRINT_ROW STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); tdSRowPrint(row, pSTSchema, __func__); @@ -2056,7 +2058,9 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu } } } - + if(rowEnd) { + tdSRowEnd(pBuilder); + } #ifdef TD_DEBUG_PRINT_ROW if (rowEnd) { STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); @@ -2436,6 +2440,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols } } + tdSRowEnd(pBuilder); pDataBlock->size += extendedRowSize; } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index ae123a3563..ee35b23cc0 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -659,6 +659,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB toffset += TYPE_BYTES[colType]; p += pSchema[j].bytes; } + tdSRowEnd(&builder); int32_t rowLen = TD_ROW_LEN((STSRow*)pDataBlock); pDataBlock = (char*)pDataBlock + rowLen; pBlock->dataLen += rowLen; From 0c4454c07764d99682d3d4d6992ba8357ef98427 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:23:27 +0800 Subject: [PATCH 14/65] fix: set statis to 0 when all col is normal --- include/common/trow.h | 2 +- source/common/src/trow.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 93004b2cbd..f2f68ddccd 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -38,7 +38,7 @@ typedef struct { uint16_t type : 2; uint16_t del : 1; uint16_t endian : 1; - uint16_t normal : 1; // all norm + uint16_t statis : 1; // 0 all normal, 1 has null or none uint16_t reserve : 11; uint16_t sver; }; diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6558af8aad..b54d653df9 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -881,7 +881,7 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, int16_t colIdx) { - if (pRow->normal) { + if (pRow->statis == 0) { if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { @@ -1097,7 +1097,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou int32_t tdSRowEnd(SRowBuilder *pBuilder) { STSRow *pRow = (STSRow *)pBuilder->pBuf; if (pBuilder->nNone || pBuilder->nNull) { - pRow->normal = 1; + pRow->statis = 1; } return TSDB_CODE_SUCCESS; } From 2b06d8328c711fdb537ac2bff21310b426311afb Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:27:50 +0800 Subject: [PATCH 15/65] enh: row end optimization --- include/common/trow.h | 7 ++++++- source/common/src/trow.c | 8 -------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index f2f68ddccd..35270341ff 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -290,7 +290,12 @@ int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen); int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, int32_t allNullLen, int32_t boundNullLen); int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); -int32_t tdSRowEnd(SRowBuilder *pBuilder); +static FORCE_INLINE void tdSRowEnd(SRowBuilder *pBuilder) { + STSRow *pRow = (STSRow *)pBuilder->pBuf; + if (pBuilder->nNone || pBuilder->nNull) { + pRow->statis = 1; + } +} int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf); void tdSRowReset(SRowBuilder *pBuilder); int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, diff --git a/source/common/src/trow.c b/source/common/src/trow.c index b54d653df9..ec51d65cdd 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -1094,14 +1094,6 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou return TSDB_CODE_SUCCESS; } -int32_t tdSRowEnd(SRowBuilder *pBuilder) { - STSRow *pRow = (STSRow *)pBuilder->pBuf; - if (pBuilder->nNone || pBuilder->nNull) { - pRow->statis = 1; - } - return TSDB_CODE_SUCCESS; -} - int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { From 22512c09222c82055038d92fd5e3581ba1fd1f78 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:33:53 +0800 Subject: [PATCH 16/65] enh: row optimization --- include/common/trow.h | 6 +++--- source/common/src/trow.c | 9 ++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 35270341ff..bd8510dde6 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -150,8 +150,8 @@ typedef struct { void *pBitmap; void *pOffset; int32_t extendedRowSize; - int16_t nNone; - int16_t nNull; + bool hasNone; + bool hasNull; } SRowBuilder; #define TD_ROW_HEAD_LEN (sizeof(STSRow)) @@ -292,7 +292,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); static FORCE_INLINE void tdSRowEnd(SRowBuilder *pBuilder) { STSRow *pRow = (STSRow *)pBuilder->pBuf; - if (pBuilder->nNone || pBuilder->nNull) { + if (pBuilder->hasNull || pBuilder->hasNone) { pRow->statis = 1; } } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index ec51d65cdd..d968221432 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -925,10 +925,10 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp case TD_VTYPE_NORM: break; case TD_VTYPE_NULL: - ++pBuilder->nNull; + if (!pBuilder->hasNull) pBuilder->hasNull = true; break; case TD_VTYPE_NONE: - ++pBuilder->nNone; + if (!pBuilder->hasNone) pBuilder->hasNone = true; break; default: ASSERT(0); @@ -1102,10 +1102,9 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { return terrno; } - if (pBuilder->nNone) pBuilder->nNone = 0; - if (pBuilder->nNull) pBuilder->nNull = 0; + if (pBuilder->hasNone) pBuilder->hasNone = false; + if (pBuilder->hasNull) pBuilder->hasNull = false; - pBuilder->nNull = 0; TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); From 258448bae9ca9b40775d47dd22fb76d7f1f0c320 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:42:08 +0800 Subject: [PATCH 17/65] fix: get normal val from tp row --- source/common/src/trow.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d968221432..a096db4b1d 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -882,6 +882,7 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, int16_t colIdx) { if (pRow->statis == 0) { + output->valType = TD_VTYPE_NORM; if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { @@ -895,6 +896,14 @@ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t return terrno; } + if (tdValTypeIsNorm(output->valType)) { + if (IS_VAR_DATA_TYPE(colType)) { + output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + } + return TSDB_CODE_SUCCESS; } From 843a516919eb5aa37add33e3cee09b9e8405e700 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:44:23 +0800 Subject: [PATCH 18/65] enh: get normal val from tp row --- source/common/src/trow.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index a096db4b1d..6e6b5acecf 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -896,7 +896,7 @@ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t return terrno; } - if (tdValTypeIsNorm(output->valType)) { + if (output->valType == TD_VTYPE_NORM) { if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { From 846ee63c88f8eb52cc6f47dd9bd01d5c3ce90f81 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:47:41 +0800 Subject: [PATCH 19/65] enh: normal val optimization --- source/common/src/trow.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6e6b5acecf..6f58a0ded1 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -912,7 +912,7 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp STSRow *pRow = pBuilder->pBuf; if (!val) { #ifdef TD_SUPPORT_BITMAP - if (tdValTypeIsNorm(valType)) { + if (valType == TD_VTYPE_NORM) { terrno = TSDB_CODE_INVALID_PTR; return terrno; } @@ -970,7 +970,7 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo STSRow *row = pBuilder->pBuf; // No need to store None/Null values. - if (tdValIsNorm(valType, val, colType)) { + if (valType == TD_VTYPE_NORM) { // ts key stored in STSRow.ts SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); @@ -1031,7 +1031,7 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo // 1. No need to set flen part for Null/None, just use bitmap. When upsert for the same primary TS key, the bitmap // should be updated simultaneously if Norm val overwrite Null/None cols. // 2. When consume STSRow in memory by taos client/tq, the output of Null/None cols should both be Null. - if (tdValIsNorm(valType, val, colType)) { + if (valType == TD_VTYPE_NORM) { // TODO: The layout of new data types imported since 3.0 like blob/medium blob is the same with binary/nchar. if (IS_VAR_DATA_TYPE(colType)) { // ts key stored in STSRow.ts From c8c9bbdeb1a6f6fa948a287be6ba3a4ec08a9222 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 3 Aug 2022 17:04:14 +0800 Subject: [PATCH 20/65] enh: last_row function scan in reverse order --- source/libs/planner/src/planOptimizer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index c02ef64707..ddf8b29153 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -242,7 +242,7 @@ static int32_t scanPathOptGetScanOrder(SScanLogicNode* pScan, EScanOrder* pScanO SFunctionNode* pFunc = (SFunctionNode*)pNode; if (FUNCTION_TYPE_FIRST == pFunc->funcType) { hasFirst = true; - } else if (FUNCTION_TYPE_LAST == pFunc->funcType) { + } else if (FUNCTION_TYPE_LAST == pFunc->funcType || FUNCTION_TYPE_LAST_ROW == pFunc->funcType) { hasLast = true; } else if (FUNCTION_TYPE_SELECT_VALUE != pFunc->funcType) { otherFunc = true; @@ -436,8 +436,8 @@ static int32_t pushDownCondOptDealScan(SOptimizeContext* pCxt, SScanLogicNode* p SNode* pPrimaryKeyCond = NULL; SNode* pOtherCond = NULL; - int32_t code = filterPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, &pScan->pTagCond, - &pOtherCond); + int32_t code = filterPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, + &pScan->pTagCond, &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pScan->pTagCond) { code = pushDownCondOptRebuildTbanme(&pScan->pTagCond); } From caa5378d8e2f2f88d64f00f53bf8ddbb711fdedd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 17:12:10 +0800 Subject: [PATCH 21/65] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b0f05e4705..93a842d24c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -296,7 +296,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; From 2a8da022def0cad9a04569ef44b72e60887a760b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 19:01:06 +0800 Subject: [PATCH 22/65] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 3 --- source/libs/function/src/builtins.c | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 93a842d24c..a6addc208d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -675,9 +675,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; -// pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose -// pInfo->cond.order = TSDB_ORDER_DESC; - pInfo->pdInfo.interval = extractIntervalInfo(pTableScanNode); pInfo->readHandle = *readHandle; pInfo->sample.sampleRatio = pTableScanNode->ratio; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 444cd2c5d0..1c519bae48 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2276,6 +2276,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = lastRowFunction, .sprocessFunc = firstLastScalarFunction, + .pPartialFunc = "_last_row_partial", + .pMergeFunc = "_last_row_merge", .finalizeFunc = firstLastFinalize }, { @@ -2288,6 +2290,27 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = cachedLastRowFunction, .finalizeFunc = firstLastFinalize }, + { + .name = "_last_row_partial", + .type = FUNCTION_TYPE_LAST_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .translateFunc = translateFirstLastPartial, + .dynDataRequiredFunc = lastDynDataReq, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastRowFunction, + .finalizeFunc = firstLastPartialFinalize, + }, + { + .name = "_last_row_merge", + .type = FUNCTION_TYPE_LAST_MERGE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .translateFunc = translateFirstLastMerge, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunctionMerge, + .finalizeFunc = firstLastFinalize, + }, { .name = "first", .type = FUNCTION_TYPE_FIRST, From c253ea73e1a6dd62870569465017ba6a77f5d8f1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 19:17:07 +0800 Subject: [PATCH 23/65] fix(query): fix the null not returns problem. --- source/libs/function/src/builtinsimpl.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 38c35e382a..a4e74e7a7d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2953,25 +2953,24 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) { SInputColumnInfoData* pColInfo = &pCtx->input; - int32_t start = pColInfo->startRowIndex; - - pOutput->bytes = pInput->bytes; - TSKEY* tsIn = &pInput->ts; - TSKEY* tsOut = &pOutput->ts; + int32_t start = pColInfo->startRowIndex; if (pOutput->hasResult) { if (isFirst) { - if (*tsIn > *tsOut) { + if (pInput->ts > pOutput->ts) { return; } } else { - if (*tsIn < *tsOut) { + if (pInput->ts < pOutput->ts) { return; } } } - *tsOut = *tsIn; + pOutput->isNull = pInput->isNull; + pOutput->ts = pInput->ts; + pOutput->bytes = pInput->bytes; + memcpy(pOutput->buf, pInput->buf, pOutput->bytes); saveTupleData(pCtx->pSrcBlock, start, pCtx, pOutput); @@ -2998,7 +2997,6 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer } SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); - return TSDB_CODE_SUCCESS; } @@ -3099,9 +3097,9 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { TSKEY startKey = getRowPTs(pInput->pPTS, 0); TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); +#if 0 int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; -#if 0 // the optimized version only function if all tuples in one block are monotonious increasing or descreasing. // this is NOT always works if project operator exists in downstream. if (blockDataOrder == TSDB_ORDER_ASC) { @@ -3141,6 +3139,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { } #endif + SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } From 0410959aacdae6b495fae1b547ecd47ee14ad9f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 19:19:17 +0800 Subject: [PATCH 24/65] test: disable some unit case. --- source/client/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 95e86e7629..f52edbe71f 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -46,6 +46,6 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) -#if(${BUILD_TEST}) +if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -#endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) \ No newline at end of file From 3566e8cf46913e7acc0e8be9266100d036e35096 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 15:02:05 +0800 Subject: [PATCH 25/65] enh(query): diff support timestamp column type. --- source/libs/function/src/builtins.c | 4 +--- source/libs/function/src/builtinsimpl.c | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1c519bae48..4ef2525b36 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1523,7 +1523,7 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType) { + if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType && TSDB_DATA_TYPE_TIMESTAMP != colType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -2204,8 +2204,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = elapsedFinalize, .invertFunc = NULL, .combineFunc = elapsedCombine, - //.pPartialFunc = "_elapsed_partial", - //.pMergeFunc = "_elapsed_merge" }, { .name = "_elapsed_partial", diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a4e74e7a7d..650aa3f12d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3170,6 +3170,8 @@ bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { switch (type) { case TSDB_DATA_TYPE_BOOL: + pDiffInfo->prev.i64 = *(bool*)pv? 1:0; + break; case TSDB_DATA_TYPE_TINYINT: pDiffInfo->prev.i64 = *(int8_t*)pv; break; @@ -3179,6 +3181,7 @@ static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { case TSDB_DATA_TYPE_SMALLINT: pDiffInfo->prev.i64 = *(int16_t*)pv; break; + case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: pDiffInfo->prev.i64 = *(int64_t*)pv; break; @@ -3232,6 +3235,7 @@ static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SCo pDiffInfo->prev.i64 = v; break; } + case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: { int64_t v = *(int64_t*)pv; int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null From 271528f4e21249c3e424a49d62634c1aed67d140 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 18:12:44 +0800 Subject: [PATCH 26/65] refactor: opt tsdb performance. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9b5523431c..7a867d85c9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2113,7 +2113,7 @@ TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pRea } TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); - TSDBKEY key = TSDBROW_KEY(pRow); + TSDBKEY key = {.ts = pRow->pTSRow->ts, .version = pRow->version}; if (outOfTimeWindow(key.ts, &pReader->window)) { pIter->hasVal = false; return NULL; From be1d1c5f8ecf6607151d8a539dcb9058e913b74f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 20:37:36 +0800 Subject: [PATCH 27/65] fix(query): clean temp files. --- source/libs/executor/src/groupoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 938134c167..528b39c17f 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -727,7 +727,7 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFree(pInfo->columnOffset); cleanupExprSupp(&pInfo->scalarSup); - + clearDiskbasedBuf(pInfo->pBuf); taosMemoryFreeClear(param); } From 27d223356a1df0a11a0c2d56801505e5c4923c3b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 4 Aug 2022 20:41:30 +0800 Subject: [PATCH 28/65] enh: merge multiple blocks in one rsp msg --- include/common/tmsg.h | 1 + include/libs/executor/dataSinkMgt.h | 1 + source/libs/qworker/inc/qwInt.h | 2 + source/libs/qworker/src/qwMsg.c | 9 ++- source/libs/qworker/src/qworker.c | 114 ++++++++++++++++++---------- 5 files changed, 83 insertions(+), 44 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bfb80ec8f8..9d001c9534 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1364,6 +1364,7 @@ typedef struct { int8_t compressed; int8_t streamBlockType; int32_t compLen; + int32_t numOfBlocks; int32_t numOfRows; int32_t numOfCols; int64_t skey; diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 47177dc11b..71105d88eb 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -67,6 +67,7 @@ typedef struct SInputData { } SInputData; typedef struct SOutputData { + int32_t numOfBlocks; int32_t numOfRows; int32_t numOfCols; int8_t compressed; diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 8f036714c9..729ac474e4 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -35,6 +35,7 @@ extern "C" { #define QW_DEFAULT_SHORT_RUN_TIMES 2 #define QW_DEFAULT_HEARTBEAT_MSEC 5000 #define QW_SCH_TIMEOUT_MSEC 180000 +#define QW_MIN_RES_ROWS 4096 enum { QW_PHASE_PRE_QUERY = 1, @@ -135,6 +136,7 @@ typedef struct SQWTaskCtx { int32_t msgType; int32_t fetchType; int32_t execId; + int32_t level; bool queryRsped; bool queryEnd; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index e8ffd98153..63a3c1bfea 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -12,14 +12,16 @@ int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { int32_t msgSize = sizeof(SRetrieveTableRsp) + length; - SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(msgSize); + SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcReallocCont(*rsp, msgSize); if (NULL == pRsp) { qError("rpcMallocCont %d failed", msgSize); QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - memset(pRsp, 0, sizeof(SRetrieveTableRsp)); - + if (NULL == *rsp) { + memset(pRsp, 0, sizeof(SRetrieveTableRsp)); + } + *rsp = pRsp; return TSDB_CODE_SUCCESS; @@ -35,6 +37,7 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) rsp->compLen = htonl(len); rsp->numOfRows = htonl(input->numOfRows); rsp->numOfCols = htonl(input->numOfCols); + rsp->numOfBlocks = htonl(input->numOfBlocks); } void qwFreeFetchRsp(void *msg) { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 36d85f1f12..d93c07ce1e 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -203,57 +203,88 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, SRetrieveTableRsp *rsp = NULL; bool queryEnd = false; int32_t code = 0; + SOutputData output = {0}; - dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); + *dataLen = 0; - if (len < 0) { - QW_TASK_ELOG("invalid length from dsGetDataLength, length:%d", len); - QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + while (true) { + dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); - if (len == 0) { - if (queryEnd) { - code = dsGetDataBlock(ctx->sinkHandle, pOutput); - if (code) { - QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); - } - - QW_TASK_DLOG_E("no data in sink and query end"); - - qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); - QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); - - *rspMsg = rsp; - *dataLen = 0; - return TSDB_CODE_SUCCESS; + if (len < 0) { + QW_TASK_ELOG("invalid length from dsGetDataLength, length:%d", len); + QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - pOutput->bufStatus = DS_BUF_EMPTY; + if (len == 0) { + if (queryEnd) { + code = dsGetDataBlock(ctx->sinkHandle, &output); + if (code) { + QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); + } - return TSDB_CODE_SUCCESS; + QW_TASK_DLOG("no more data in sink and query end, fetched blocks %d rows %d", pOutput->numOfBlocks, pOutput->numOfRows); + + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); + if (NULL == rsp) { + QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); + *pOutput = output; + } else { + pOutput->queryEnd = output.queryEnd; + pOutput->bufStatus = output.bufStatus; + pOutput->useconds = output.useconds; + } + + break; + } + + pOutput->bufStatus = DS_BUF_EMPTY; + + break; + } + + // Got data from sink + QW_TASK_DLOG("there are data in sink, dataLength:%d", len); + + *dataLen += len; + + QW_ERR_RET(qwMallocFetchRsp(*dataLen, &rsp)); + + output.pData = rsp->data + *dataLen - len; + code = dsGetDataBlock(ctx->sinkHandle, &output); + if (code) { + QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); + } + + pOutput->queryEnd = output.queryEnd; + pOutput->precision = output.precision; + pOutput->bufStatus = output.bufStatus; + pOutput->useconds = output.useconds; + pOutput->compressed = output.compressed; + pOutput->numOfCols = output.numOfCols; + pOutput->numOfRows += output.numOfRows; + pOutput->numOfBlocks++; + + if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { + QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %d", pOutput->numOfBlocks, pOutput->numOfRows); + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); + break; + } + + if (0 == ctx->level) { + QW_TASK_DLOG("task fetched blocks %d rows %d, level %d", pOutput->numOfBlocks, pOutput->numOfRows, ctx->level); + break; + } + + if (pOutput->numOfRows >= QW_MIN_RES_ROWS) { + QW_TASK_DLOG("task fetched blocks %d rows %d reaches the min rows", pOutput->numOfBlocks, pOutput->numOfRows); + break; + } } - // Got data from sink - QW_TASK_DLOG("there are data in sink, dataLength:%d", len); - - *dataLen = len; - - QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); *rspMsg = rsp; - pOutput->pData = rsp->data; - code = dsGetDataBlock(ctx->sinkHandle, pOutput); - if (code) { - QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); - } - - if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { - QW_TASK_DLOG_E("task all data fetched, done"); - qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); - } - return TSDB_CODE_SUCCESS; } @@ -551,6 +582,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) { // queryRsped = true; + ctx->level = plan->level; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); atomic_store_ptr(&ctx->sinkHandle, sinkHandle); From 569c82886f4d26488a896b43e785004b64202b56 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 20:42:23 +0800 Subject: [PATCH 29/65] fix(query): remove temp files. --- source/libs/executor/src/groupoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 528b39c17f..507719e0aa 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -727,7 +727,7 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFree(pInfo->columnOffset); cleanupExprSupp(&pInfo->scalarSup); - clearDiskbasedBuf(pInfo->pBuf); + destroyDiskbasedBuf(pInfo->pBuf); taosMemoryFreeClear(param); } From 0e4748771acd90aceeee5141b094bbce64e7d1ed Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 14:10:45 +0800 Subject: [PATCH 30/65] enh(query): support combine multi datablock to pass to upstream operators --- include/common/tdatablock.h | 2 +- source/client/src/clientImpl.c | 11 ++ source/common/src/tdatablock.c | 34 +++++- source/dnode/vnode/src/tsdb/tsdbRead.c | 7 +- source/libs/executor/inc/executorimpl.h | 9 +- source/libs/executor/src/dataDispatcher.c | 8 +- source/libs/executor/src/executorimpl.c | 142 +++++++++++++--------- source/libs/executor/src/scanoperator.c | 4 +- source/libs/stream/src/streamData.c | 5 +- 9 files changed, 142 insertions(+), 80 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 7839859e8b..3d605a144b 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -239,7 +239,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); -const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); +const char* blockDecode(SSDataBlock* pBlock, const char* pData); void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag); void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0c4dc1705c..8357e2d627 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1765,6 +1765,17 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t dataLen = *(int32_t*)p; p += sizeof(int32_t); + int32_t rows = *(int32_t*)p; + p += sizeof(int32_t); + + int32_t cols = *(int32_t*)p; + p += sizeof(int32_t); + + ASSERT(rows == numOfRows && cols == numOfCols); + + int32_t hasColumnSeg = *(int32_t*)p; + p += sizeof(int32_t); + uint64_t groupId = *(uint64_t*)p; p += sizeof(uint64_t); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4ce60efaea..bcbdf8d786 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -676,9 +676,9 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { * @return */ size_t blockDataGetSerialMetaSize(uint32_t numOfCols) { - // | total rows/total length | block group id | column schema | each column length | - return sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + - numOfCols * sizeof(int32_t); + // | total length | total rows | total columns | has column seg| block group id | column schema | each column length | + return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); } double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { @@ -2077,6 +2077,18 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ int32_t* actualLen = (int32_t*)data; data += sizeof(int32_t); + int32_t* rows = (int32_t*)data; + *rows = pBlock->info.rows; + data += sizeof(int32_t); + + int32_t* cols = (int32_t*)data; + *cols = numOfCols; + data += sizeof(int32_t); + + int32_t* hasColumnSegment = (int32_t*)data; + *hasColumnSegment = 1; + data += sizeof(int32_t); + uint64_t* groupId = (uint64_t*)data; data += sizeof(uint64_t); @@ -2130,12 +2142,26 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ *groupId = pBlock->info.groupId; } -const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { +const char* blockDecode(SSDataBlock* pBlock, const char* pData) { const char* pStart = pData; + // total length sizeof(int32_t) int32_t dataLen = *(int32_t*)pStart; pStart += sizeof(int32_t); + // total rows sizeof(int32_t) + int32_t numOfRows = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + // total columns sizeof(int32_t) + int32_t numOfCols = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + // has column info segment + int32_t hasColumnInfo = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + // group id sizeof(uint64_t) pBlock->info.groupId = *(uint64_t*)pStart; pStart += sizeof(uint64_t); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7a867d85c9..2a8e234ac0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2714,7 +2714,6 @@ void tsdbReaderClose(STsdbReader* pReader) { } SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap); taosMemoryFreeClear(pSupInfo->plist); @@ -2742,10 +2741,8 @@ void tsdbReaderClose(STsdbReader* pReader) { SIOCostSummary* pCost = &pReader->cost; tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 - " SMA-time:%.2f ms, " - "fileBlocks:%" PRId64 - ", fileBlocks-time:%.2f ms, build in-memory-block-time:%.2f ms, STableBlockScanInfo " - "size:%.2f Kb %s", + " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-time:%.2f ms, " + "build in-memory-block-time:%.2f ms, STableBlockScanInfo size:%.2f Kb %s", pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaData, pCost->smaLoadTime, pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 2e0ed802e5..62b2cc0fa6 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -266,7 +266,11 @@ typedef struct SExchangeInfo { SArray* pSourceDataInfo; tsem_t ready; void* pTransporter; - SSDataBlock* pResult; + // SArray, result block list, used to keep the multi-block that + // passed by downstream operator + SArray* pResultBlockList; + int32_t rspBlockIndex; // indicate the return block index in pResultBlockList + SSDataBlock* pDummyBlock; // dummy block, not keep data bool seqLoadData; // sequential load data or not, false by default int32_t current; SLoadRemoteDataInfo loadInfo; @@ -855,8 +859,7 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWin int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, - SArray* pColList); + int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart); STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index f0fb5852a0..3a37b5e760 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -69,10 +69,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { // clang-format off // data format: -// +----------------+--------------+-----------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) -// | |sizeof(int32) |sizeof(uint64_t) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | -// +----------------+--------------+-----------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// |SDataCacheEntry | total length | numOfRows | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) +// | |sizeof(int32) | sizeof(int32)| sizeof(uint64_t) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | +// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header // clang-format on diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 23dbd4d992..8dfda896cc 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1956,6 +1956,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->compLen = htonl(pRsp->compLen); pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); + pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); qDebug("%s fetch rsp received, index:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfRows); @@ -2043,12 +2044,10 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf } int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, - SArray* pColList) { + int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart) { if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); - // blockDataEnsureCapacity(pRes, numOfRows); - blockDecode(pRes, numOfOutput, numOfRows, pData); + *pNextStart = (char*) blockDecode(pRes, pData); } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; @@ -2072,7 +2071,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo blockDataAppendColInfo(pBlock, &idata); } - blockDecode(pBlock, numOfCols, numOfRows, pStart); + blockDecode(pBlock, pStart); blockDataEnsureCapacity(pRes, numOfRows); // data from mnode @@ -2084,8 +2083,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. blockDataUpdateTsWindow(pRes, 0); - int64_t el = taosGetTimestampUs() - startTs; - pLoadInfo->totalRows += numOfRows; pLoadInfo->totalSize += compLen; @@ -2093,7 +2090,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo *total += numOfRows; } - pLoadInfo->totalElapsed += el; return TSDB_CODE_SUCCESS; } @@ -2115,8 +2111,8 @@ static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { return NULL; } -static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, - SExecTaskInfo* pTaskInfo) { +static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, + SExecTaskInfo* pTaskInfo) { int32_t code = 0; int64_t startTs = taosGetTimestampUs(); size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); @@ -2142,7 +2138,6 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx SRetrieveTableRsp* pRsp = pDataInfo->pRsp; SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); - SSDataBlock* pRes = pExchangeInfo->pResult; SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pRsp->numOfRows == 0) { qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 @@ -2155,29 +2150,37 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx continue; } - SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; - code = - extractDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); - if (code != 0) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + int32_t index = 0; + char* pStart = pRetrieveRsp->data; + while(index++ < pRetrieveRsp->numOfBlocks) { + SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); + blockDataEnsureCapacity(pb, pRetrieveRsp->numOfRows); + code = + extractDataBlockFromFetchRsp(pb, pLoadInfo, pRetrieveRsp->numOfRows, pStart, + pRetrieveRsp->compLen, pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); + if (code != 0) { + taosMemoryFreeClear(pDataInfo->pRsp); + goto _error; + } + + taosArrayPush(pExchangeInfo->pResultBlockList, &pb); } + pLoadInfo->totalElapsed += (taosGetTimestampUs() - startTs); + if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d" - " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 - ", completed:%d try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources); + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d" + " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb," + " completed:%d try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfRows, + pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0, completed + 1, i + 1, totalSources); completed += 1; pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 - ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, - pLoadInfo->totalRows, pLoadInfo->totalSize); + ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, + pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0); } taosMemoryFreeClear(pDataInfo->pRsp); @@ -2191,11 +2194,12 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx } } - return pExchangeInfo->pResult; + return; } if (completed == totalSources) { - return setAllSourcesCompleted(pOperator, startTs); + setAllSourcesCompleted(pOperator, startTs); + return; } sched_yield(); @@ -2203,7 +2207,6 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx _error: pTaskInfo->code = code; - return NULL; } static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { @@ -2233,7 +2236,7 @@ static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { +static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2242,7 +2245,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { while (1) { if (pExchangeInfo->current >= totalSources) { - return setAllSourcesCompleted(pOperator, startTs); + setAllSourcesCompleted(pOperator, startTs); + return TSDB_CODE_SUCCESS; } doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); @@ -2255,7 +2259,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); pOperator->pTaskInfo->code = pDataInfo->code; - return NULL; + return pOperator->pTaskInfo->code; } SRetrieveTableRsp* pRsp = pDataInfo->pRsp; @@ -2272,16 +2276,16 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { continue; } - SSDataBlock* pRes = pExchangeInfo->pResult; - SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; - int32_t code = - extractDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + + char* pStart = pRetrieveRsp->data; + int32_t code = extractDataBlockFromFetchRsp(NULL, pLoadInfo, pRetrieveRsp->numOfRows, pStart, pRetrieveRsp->compLen, + pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources); @@ -2290,13 +2294,13 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize); } - pOperator->resultInfo.totalRows += pRes->info.rows; + pOperator->resultInfo.totalRows += pRetrieveRsp->numOfRows; taosMemoryFreeClear(pDataInfo->pRsp); - return pExchangeInfo->pResult; + return TSDB_CODE_SUCCESS; } } @@ -2320,6 +2324,11 @@ static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } +static void freeBlock(void* pParam) { + SSDataBlock* pBlock = *(SSDataBlock**)pParam; + blockDataDestroy(pBlock); +} + static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2329,9 +2338,9 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { return NULL; } - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pOperator->status == OP_EXEC_DONE) { qDebug("%s all %" PRIzu " source(s) are exhausted, total rows:%" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, @@ -2339,11 +2348,23 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { return NULL; } - if (pExchangeInfo->seqLoadData) { - return seqLoadRemoteData(pOperator); - } else { - return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList); + if (size == 0 || pExchangeInfo->rspBlockIndex >= size) { + pExchangeInfo->rspBlockIndex = 0; + taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock); + if (pExchangeInfo->seqLoadData) { + seqLoadRemoteData(pOperator); + } else { + concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + } + + if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { + return NULL; + } } + + // we have buffered retrieved datablock, return it directly + return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++); } static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { @@ -2360,26 +2381,24 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { return NULL; } - ASSERT(pBlock == pExchangeInfo->pResult); - SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; if (hasLimitOffsetInfo(pLimitInfo)) { - int32_t status = handleLimitOffset(pOperator, pLimitInfo, pExchangeInfo->pResult, false); + int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); if (status == PROJECT_RETRIEVE_CONTINUE) { continue; } else if (status == PROJECT_RETRIEVE_DONE) { - size_t rows = pExchangeInfo->pResult->info.rows; + size_t rows = pBlock->info.rows; pExchangeInfo->limitInfo.numOfOutputRows += rows; if (rows == 0) { doSetOperatorCompleted(pOperator); return NULL; } else { - return pExchangeInfo->pResult; + return pBlock; } } } else { - return pExchangeInfo->pResult; + return pBlock; } } } @@ -2442,16 +2461,18 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode } tsem_init(&pInfo->ready, 0, 0); + pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES); pInfo->seqLoadData = false; pInfo->pTransporter = pTransporter; - pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pResult->pDataBlock); + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, @@ -3585,12 +3606,15 @@ void doDestroyExchangeOperatorInfo(void* param) { taosArrayDestroy(pExInfo->pSources); taosArrayDestroy(pExInfo->pSourceDataInfo); - if (pExInfo->pResult != NULL) { - pExInfo->pResult = blockDataDestroy(pExInfo->pResult); + + if (pExInfo->pResultBlockList != NULL) { + taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock); + pExInfo->pResultBlockList = NULL; } - tsem_destroy(&pExInfo->ready); + blockDataDestroy(pExInfo->pDummyBlock); + tsem_destroy(&pExInfo->ready); taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8545abecfb..03c08a6bf5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2266,9 +2266,11 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } } + char* pStart = pRsp->data; extractDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pRsp->numOfRows, pRsp->data, pRsp->compLen, - pOperator->exprSupp.numOfExprs, startTs, NULL, pInfo->scanCols); + pOperator->exprSupp.numOfExprs, NULL, pInfo->scanCols, &pStart); + //startTs, // todo log the filter info doFilterResult(pInfo); taosMemoryFree(pRsp); diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 0bf6d4c921..c96854b198 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -27,10 +27,9 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock ASSERT(pReq->blockNum == taosArrayGetSize(pReq->dataLen)); for (int32_t i = 0; i < blockNum; i++) { - /*int32_t len = *(int32_t*)taosArrayGet(pReq->dataLen, i);*/ SRetrieveTableRsp* pRetrieve = taosArrayGetP(pReq->data, i); SSDataBlock* pDataBlock = taosArrayGet(pArray, i); - blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); + blockDecode(pDataBlock, pRetrieve->data); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); @@ -51,7 +50,7 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock taosArraySetSize(pArray, 1); SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); - blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); + blockDecode(pDataBlock, pRetrieve->data); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); From d0524389d9501723f48514255636358e437a90ce Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 5 Aug 2022 15:07:38 +0800 Subject: [PATCH 31/65] fix: plan problem with interval fill statement --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 1ebba8bb63..2e2c4fbff7 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -719,7 +719,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; - pWindow->node.groupAction = getGroupAction(pCxt, pSelect); + pWindow->node.groupAction = (NULL != pInterval->pFill ? GROUP_ACTION_KEEP : getGroupAction(pCxt, pSelect)); pWindow->node.requireDataOrder = pSelect->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_IN_BLOCK; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; From 0ac1b85a0e72b399a08f56bbefd25c9634c8a4e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 15:09:43 +0800 Subject: [PATCH 32/65] refactor: add some logs. --- source/libs/executor/src/executorimpl.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8dfda896cc..d17fe7633f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1959,7 +1959,8 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfRows); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows); } else { pSourceDataInfo->code = code; qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); @@ -2171,16 +2172,17 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d" - " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb," + " index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb," " completed:%d try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfRows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0, completed + 1, i + 1, totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, + pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, + completed + 1, i + 1, totalSources); completed += 1; pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, - pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0); + pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0); } taosMemoryFreeClear(pDataInfo->pRsp); From 52173e14e1b753f4a530f205f05024b01e89ff55 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 16:10:45 +0800 Subject: [PATCH 33/65] fix(query): add check for null ptr. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2a8e234ac0..0749d7b133 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2312,6 +2312,12 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe return; } else { // has next point in mem/imem TSDBROW* pNextRow = getValidRow(pIter, pDelList, pReader); + if (pNextRow == NULL) { + *pTSRow = current.pTSRow; + *freeTSRow = false; + return; + } + if (TSDBROW_KEY(¤t).ts != TSDBROW_KEY(pNextRow).ts) { *pTSRow = current.pTSRow; *freeTSRow = false; From 1063064a59916219b24605d0b89ddc02552ad747 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 17:40:40 +0800 Subject: [PATCH 34/65] enh(query): support filter by SMA. --- include/libs/scalar/filter.h | 2 +- source/libs/executor/src/executorimpl.c | 12 +-- source/libs/executor/src/scanoperator.c | 116 ++++++++++++++++-------- source/libs/scalar/src/filter.c | 12 +-- 4 files changed, 84 insertions(+), 58 deletions(-) diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index 44064f0a9f..1f1d9dea93 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -44,7 +44,7 @@ extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo); extern void filterFreeInfo(SFilterInfo *info); -extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); +extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t numOfCols, int32_t numOfRows); /* condition split interface */ int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d17fe7633f..1908e2a92d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1011,16 +1011,6 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { // } //} -// static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, -// SqlFunctionCtx *pCtx, int32_t numOfRows) { -// STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -// -// if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { -// return true; -// } -// -// return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows); -// } #if 0 static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { STimeWindow w = {0}; @@ -1215,7 +1205,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc } // current block has been discard due to filter applied -// if (!doFilterByBlockStatistics(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) { +// if (!doFilterByBlockSMA(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) { // pCost->skipBlocks += 1; // qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, // pBlockInfo->window.ekey, pBlockInfo->rows); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 03c08a6bf5..a2e0b68623 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,10 +13,11 @@ * along with this program. If not, see . */ +#include "os.h" #include "executorimpl.h" +#include "filter.h" #include "function.h" #include "functionMgt.h" -#include "os.h" #include "querynodes.h" #include "systable.h" #include "tname.h" @@ -227,6 +228,57 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* return TSDB_CODE_SUCCESS; } +static FORCE_INLINE bool doFilterByBlockSMA(const SNode* pFilterNode, SColumnDataAgg** pColsAgg, int32_t numOfCols, + int32_t numOfRows) { + if (pColsAgg == NULL || pFilterNode == NULL) { + return true; + } + + SFilterInfo* filter = NULL; + + // todo move to the initialization function + int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + bool keep = filterRangeExecute(filter, pColsAgg, numOfCols, numOfRows); + + filterFreeInfo(filter); + return keep; +} + +static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + bool allColumnsHaveAgg = true; + SColumnDataAgg** pColAgg = NULL; + + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + if (!allColumnsHaveAgg) { + return false; + } + + // if (allColumnsHaveAgg == true) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + + // todo create this buffer during creating operator + if (pBlock->pBlockAgg == NULL) { + pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); + if (pBlock->pBlockAgg == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + } + } + + for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->pColMatchInfo); ++i) { + SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); + if (!pColMatchInfo->output) { + continue; + } + pBlock->pBlockAgg[pColMatchInfo->targetSlotId] = pColAgg[i]; + } + + return true; +} + static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -236,6 +288,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pCost->totalBlocks += 1; pCost->totalRows += pBlock->info.rows; + bool loadSMA = false; *status = pInfo->dataBlockLoadFlag; if (pTableScanInfo->pFilterNode != NULL || @@ -259,41 +312,34 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; - - bool allColumnsHaveAgg = true; - SColumnDataAgg** pColAgg = NULL; - - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } - - if (allColumnsHaveAgg == true) { - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - - // todo create this buffer during creating operator - if (pBlock->pBlockAgg == NULL) { - pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); - } - - for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->pColMatchInfo); ++i) { - SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); - if (!pColMatchInfo->output) { - continue; - } - pBlock->pBlockAgg[pColMatchInfo->targetSlotId] = pColAgg[i]; - } - - return TSDB_CODE_SUCCESS; - } else { // failed to load the block sma data, data block statistics does not exist, load data block instead + loadSMA = true; // mark the operator of load sma; + bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); + if (!success) { // failed to load the block sma data, data block statistics does not exist, load data block instead *status = FUNC_DATA_REQUIRED_DATA_LOAD; } } ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD); - // todo filter data block according to the block sma data firstly + // try to filter data block according to sma info + if (pTableScanInfo->pFilterNode != NULL) { + if (!loadSMA) { + doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); + } + bool keep = doFilterByBlockSMA(pTableScanInfo->pFilterNode, pBlock->pBlockAgg, taosArrayGetSize(pBlock->pDataBlock), + pBlockInfo->rows); + if (!keep) { + qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + pCost->filterOutBlocks += 1; + (*status) = FUNC_DATA_REQUIRED_FILTEROUT; + + return TSDB_CODE_SUCCESS; + } + } + + // try to filter datablock according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), @@ -303,16 +349,6 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return TSDB_CODE_SUCCESS; } -#if 0 - if (!doFilterByBlockStatistics(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { - pCost->filterOutBlocks += 1; - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); - (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - return TSDB_CODE_SUCCESS; - } -#endif - pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; @@ -2722,7 +2758,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc // todo filter data block according to the block sma data firstly #if 0 - if (!doFilterByBlockStatistics(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { + if (!doFilterByBlockSMA(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { pCost->filterOutBlocks += 1; qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 04328fda9c..a6a47da2e5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3245,7 +3245,7 @@ _return: return code; } -bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows) { +bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { if (FILTER_EMPTY_RES(info)) { return false; } @@ -3261,7 +3261,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t int32_t index = -1; SFilterRangeCtx *ctx = info->colRange[k]; for(int32_t i = 0; i < numOfCols; ++i) { - if (pDataStatis[i].colId == ctx->colId) { + if (pDataStatis[i]->colId == ctx->colId) { index = i; break; } @@ -3277,13 +3277,13 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t break; } - if (pDataStatis[index].numOfNull <= 0) { + if (pDataStatis[index]->numOfNull <= 0) { if (ctx->isnull && !ctx->notnull && !ctx->isrange) { ret = false; break; } - } else if (pDataStatis[index].numOfNull > 0) { - if (pDataStatis[index].numOfNull == numOfRows) { + } else if (pDataStatis[index]->numOfNull > 0) { + if (pDataStatis[index]->numOfNull == numOfRows) { if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { ret = false; break; @@ -3297,7 +3297,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t } } - SColumnDataAgg* pDataBlockst = &pDataStatis[index]; + SColumnDataAgg* pDataBlockst = pDataStatis[index]; SFilterRangeNode *r = ctx->rs; float minv = 0; From c7b511ed698b8addf86dc822d29ce961ba8e020e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 6 Aug 2022 16:13:18 +0800 Subject: [PATCH 35/65] fix(query): set src block for function. --- source/libs/executor/src/executorimpl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1908e2a92d..14914b51c0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -434,6 +434,7 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC pCtx[i].order = order; pCtx[i].input.numOfRows = pBlock->info.rows; setBlockStatisInfo(&pCtx[i], &pOperator->exprSupp.pExprInfo[i], pBlock); + pCtx[i].pSrcBlock = pBlock; } } From ebe8192d7ecd526546c0eab9a5c11087de5bda88 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 6 Aug 2022 16:15:15 +0800 Subject: [PATCH 36/65] refactor(query): do some internal refactor. --- source/libs/executor/src/executorimpl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 14914b51c0..b0929a9acd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -88,7 +88,7 @@ static int32_t getExprFunctionId(SExprInfo* pExprInfo) { static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); -static void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pSDataBlock); +static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pSDataBlock); static void releaseQueryBuf(size_t numOfTables); @@ -433,7 +433,7 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { pCtx[i].order = order; pCtx[i].input.numOfRows = pBlock->info.rows; - setBlockStatisInfo(&pCtx[i], &pOperator->exprSupp.pExprInfo[i], pBlock); + setBlockSMAInfo(&pCtx[i], &pOperator->exprSupp.pExprInfo[i], pBlock); pCtx[i].pSrcBlock = pBlock; } } @@ -799,7 +799,7 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF return TSDB_CODE_SUCCESS; } -void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pBlock) { +void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pBlock) { int32_t numOfRows = pBlock->info.rows; SInputColumnInfoData* pInput = &pCtx->input; From 5ae9fdff7a59bee3c685c147d142235650335770 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 8 Aug 2022 10:00:10 +0800 Subject: [PATCH 37/65] enh: increase explain output, enhance debugging function --- source/libs/command/inc/commandInt.h | 5 +++ source/libs/command/src/explain.c | 31 +++++++++++++++-- source/libs/function/src/builtins.c | 7 ++-- source/libs/parser/src/parInsertData.c | 6 ++-- source/libs/parser/src/parTranslater.c | 2 -- source/libs/planner/src/planLogicCreater.c | 40 ++++++++++++++-------- source/libs/planner/src/planPhysiCreater.c | 2 +- 7 files changed, 67 insertions(+), 26 deletions(-) diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index c53d464eab..e2288ed3ef 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -43,6 +43,7 @@ extern "C" { #define EXPLAIN_GROUP_SORT_FORMAT "Group Sort" #define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s" #define EXPLAIN_MERGE_INTERVAL_FORMAT "Merge Interval on Column %s" +#define EXPLAIN_MERGE_ALIGNED_INTERVAL_FORMAT "Merge Aligned Interval on Column %s" #define EXPLAIN_FILL_FORMAT "Fill" #define EXPLAIN_SESSION_FORMAT "Session" #define EXPLAIN_STATE_WINDOW_FORMAT "StateWindow on Column %s" @@ -59,6 +60,8 @@ extern "C" { #define EXPLAIN_RATIO_TIME_FORMAT "Ratio: %f" #define EXPLAIN_MERGE_FORMAT "SortMerge" #define EXPLAIN_MERGE_KEYS_FORMAT "Merge Key: " +#define EXPLAIN_PARTITION_KETS_FORMAT "Partition Key: " +#define EXPLAIN_INTERP_FORMAT "Interp" #define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms" #define EXPLAIN_EXEC_TIME_FORMAT "Execution Time: %.3f ms" @@ -69,6 +72,7 @@ extern "C" { #define EXPLAIN_LEFT_PARENTHESIS_FORMAT " (" #define EXPLAIN_RIGHT_PARENTHESIS_FORMAT ")" #define EXPLAIN_BLANK_FORMAT " " +#define EXPLAIN_COMMA_FORMAT ", " #define EXPLAIN_COST_FORMAT "cost=%.2f..%.2f" #define EXPLAIN_ROWS_FORMAT "rows=%" PRIu64 #define EXPLAIN_COLUMNS_FORMAT "columns=%d" @@ -86,6 +90,7 @@ extern "C" { #define EXPLAIN_OUTPUT_ORDER_TYPE_FORMAT "output_order=%s" #define EXPLAIN_OFFSET_FORMAT "offset=%d" #define EXPLAIN_SOFFSET_FORMAT "soffset=%d" +#define EXPLAIN_PARTITIONS_FORMAT "partitions=%d" #define COMMAND_RESET_LOG "resetLog" #define COMMAND_SCHEDULE_POLICY "schedulePolicy" diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 2836bd7acd..5935b19173 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -535,6 +535,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (NULL != pTblScanNode->pGroupTags) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PARTITION_KETS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_PARTITIONS_FORMAT, pTblScanNode->pGroupTags->length); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, @@ -612,13 +619,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pPrjNode->mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pPrjNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -915,7 +926,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: { SMergeAlignedIntervalPhysiNode *pIntNode = (SMergeAlignedIntervalPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); + EXPLAIN_ROW_NEW(level, EXPLAIN_MERGE_ALIGNED_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); @@ -1129,6 +1140,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PARTITION_KETS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_PARTITIONS_FORMAT, pPartNode->pPartitionKeys->length); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pPartNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pPartNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, @@ -1201,11 +1217,20 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGE_KEYS_FORMAT); + if (pMergeNode->groupSort) { + EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, "_group_id asc"); + if (LIST_LENGTH(pMergeNode->pMergeKeys) > 0) { + EXPLAIN_ROW_APPEND(EXPLAIN_COMMA_FORMAT); + } + } for (int32_t i = 0; i < LIST_LENGTH(pMergeNode->pMergeKeys); ++i) { SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pMergeNode->pMergeKeys, i); EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, nodesGetNameFromColumnNode(ptn->pExpr)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, EXPLAIN_ORDER_STRING(ptn->order)); + if (i != LIST_LENGTH(pMergeNode->pMergeKeys) - 1) { + EXPLAIN_ROW_APPEND(EXPLAIN_COMMA_FORMAT); + } } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -1417,7 +1442,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: { SInterpFuncPhysiNode *pInterpNode = (SInterpFuncPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT); + EXPLAIN_ROW_NEW(level, EXPLAIN_INTERP_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bdd9232ce3..13ebcd4518 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1575,7 +1575,8 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType && TSDB_DATA_TYPE_TIMESTAMP != colType) { + if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType && + TSDB_DATA_TYPE_TIMESTAMP != colType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -2150,7 +2151,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "_apercentile_partial", - .type = FUNCTION_TYPE_APERCENTILE_PARTIAL | FUNC_MGT_TIMELINE_FUNC, + .type = FUNCTION_TYPE_APERCENTILE_PARTIAL, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateApercentilePartial, .getEnvFunc = getApercentileFuncEnv, @@ -2163,7 +2164,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_apercentile_merge", .type = FUNCTION_TYPE_APERCENTILE_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateApercentileMerge, .getEnvFunc = getApercentileFuncEnv, .initFunc = apercentileFunctionSetup, diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index ee35b23cc0..cd30672baf 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -486,11 +486,11 @@ static int32_t tdBlockRowMerge(STableMeta* pTableMeta, SBlockKeyTuple* pEndKeyTp void* pDestRow = NULL; ++((*pBlkRowMerger)->index); if ((*pBlkRowMerger)->index < taosArrayGetSize((*pBlkRowMerger)->rowArray)) { - void* pAlloc = *(void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index); - if (tRealloc((uint8_t**)&pAlloc, rowSize) != 0) { + void** pAlloc = (void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index); + if (tRealloc((uint8_t**)pAlloc, rowSize) != 0) { return TSDB_CODE_FAILED; } - pDestRow = pAlloc; + pDestRow = *pAlloc; } else { if (tRealloc((uint8_t**)&pDestRow, rowSize) != 0) { return TSDB_CODE_FAILED; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d96cdca8a9..2f6725acaf 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2543,7 +2543,6 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } - pSelect->isTimeLineResult = true; pCxt->currClause = SQL_CLAUSE_WINDOW; int32_t code = translateExpr(pCxt, &pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { @@ -2614,7 +2613,6 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec if (NULL == pSelect->pPartitionByList) { return TSDB_CODE_SUCCESS; } - pSelect->isTimeLineResult = false; pCxt->currClause = SQL_CLAUSE_PARTITION_BY; return translateExprList(pCxt, pSelect->pPartitionByList); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 2e2c4fbff7..98b97765de 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -342,7 +342,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pJoin->inputTsOrder = ORDER_ASC; pJoin->node.groupAction = GROUP_ACTION_CLEAR; pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; - pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; + pJoin->node.resultDataOrder = DATA_ORDER_LEVEL_GLOBAL; int32_t code = TSDB_CODE_SUCCESS; @@ -472,6 +472,11 @@ static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; } +static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { + return needTimeline ? (NULL != pSelect->pPartitionByList ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) + : DATA_ORDER_LEVEL_NONE; +} + static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; @@ -486,8 +491,8 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->hasTimeLineFunc = pSelect->hasTimeLineFunc; pAgg->onlyHasKeepOrderFunc = pSelect->onlyHasKeepOrderFunc; pAgg->node.groupAction = getGroupAction(pCxt, pSelect); - pAgg->node.requireDataOrder = pAgg->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; - pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; + pAgg->node.requireDataOrder = getRequireDataOrder(pAgg->hasTimeLineFunc, pSelect); + pAgg->node.resultDataOrder = pAgg->onlyHasKeepOrderFunc ? pAgg->node.requireDataOrder : DATA_ORDER_LEVEL_NONE; int32_t code = TSDB_CODE_SUCCESS; @@ -557,8 +562,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt pIdfRowsFunc->isUniqueFunc = pSelect->hasUniqueFunc; pIdfRowsFunc->isTimeLineFunc = pSelect->hasTimeLineFunc; pIdfRowsFunc->node.groupAction = getGroupAction(pCxt, pSelect); - pIdfRowsFunc->node.requireDataOrder = - pIdfRowsFunc->isTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; + pIdfRowsFunc->node.requireDataOrder = getRequireDataOrder(pIdfRowsFunc->isTimeLineFunc, pSelect); pIdfRowsFunc->node.resultDataOrder = pIdfRowsFunc->node.requireDataOrder; // indefinite rows functions and _select_values functions @@ -592,7 +596,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p } pInterpFunc->node.groupAction = getGroupAction(pCxt, pSelect); - pInterpFunc->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pInterpFunc->node.requireDataOrder = getRequireDataOrder(true, pSelect); pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder; int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsInterpFunc, &pInterpFunc->pFuncs); @@ -668,8 +672,10 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo pWindow->winType = WINDOW_TYPE_STATE; pWindow->node.groupAction = getGroupAction(pCxt, pSelect); - pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; - pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : getRequireDataOrder(true, pSelect); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : pWindow->node.requireDataOrder; pWindow->pStateExpr = nodesCloneNode(pState->pExpr); pWindow->pTspk = nodesCloneNode(pState->pCol); if (NULL == pWindow->pStateExpr || NULL == pWindow->pTspk) { @@ -691,8 +697,10 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i; pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? SESSION_ALGO_STREAM_SINGLE : SESSION_ALGO_MERGE; pWindow->node.groupAction = getGroupAction(pCxt, pSelect); - pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; - pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : getRequireDataOrder(true, pSelect); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : pWindow->node.requireDataOrder; pWindow->pTspk = nodesCloneNode((SNode*)pSession->pCol); if (NULL == pWindow->pTspk) { @@ -720,8 +728,12 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; pWindow->node.groupAction = (NULL != pInterval->pFill ? GROUP_ACTION_KEEP : getGroupAction(pCxt, pSelect)); - pWindow->node.requireDataOrder = pSelect->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_IN_BLOCK; - pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery + ? DATA_ORDER_LEVEL_IN_BLOCK + : (pSelect->hasTimeLineFunc ? getRequireDataOrder(true, pSelect) : DATA_ORDER_LEVEL_IN_BLOCK); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : getRequireDataOrder(true, pSelect); pWindow->pTspk = nodesCloneNode(pInterval->pCol); if (NULL == pWindow->pTspk) { @@ -768,8 +780,8 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } pFill->node.groupAction = getGroupAction(pCxt, pSelect); - pFill->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; - pFill->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pFill->node.requireDataOrder = getRequireDataOrder(true, pSelect); + pFill->node.resultDataOrder = pFill->node.requireDataOrder; pFill->inputTsOrder = ORDER_ASC; int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_WINDOW, NULL, COLLECT_COL_TYPE_ALL, &pFill->node.pTargets); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 4fe229269a..ff2e2f1494 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -985,7 +985,7 @@ static bool projectCanMergeDataBlock(SProjectLogicNode* pProject) { return true; } if (1 != LIST_LENGTH(pProject->node.pChildren)) { - return false; + return true; } SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProject->node.pChildren, 0); return DATA_ORDER_LEVEL_GLOBAL == pChild->resultDataOrder ? true : false; From 447b2cb9f1b8cd84e6e1c75890488c0ee9367e3d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Aug 2022 12:56:03 +0800 Subject: [PATCH 38/65] fix(query): update the load data log. --- source/libs/executor/inc/executorimpl.h | 6 ++-- source/libs/executor/src/executorimpl.c | 37 ++++++++++++------------- source/libs/executor/src/scanoperator.c | 5 ++-- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 62b2cc0fa6..1b8be3b53c 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -858,8 +858,10 @@ void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLi void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart); +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, char** pNextStart); +void updateLoadRemoteInfo(SLoadRemoteDataInfo *pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator); + STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b0929a9acd..752b93bcea 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2035,8 +2035,16 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf return TSDB_CODE_SUCCESS; } -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart) { +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator) { + pInfo->totalRows += numOfRows; + pInfo->totalSize += dataLen; + pInfo->totalElapsed += (taosGetTimestampUs() - startTs); + pOperator->resultInfo.totalRows += numOfRows; +} + +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, + char** pNextStart) { if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); *pNextStart = (char*) blockDecode(pRes, pData); @@ -2064,24 +2072,16 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo } blockDecode(pBlock, pStart); - blockDataEnsureCapacity(pRes, numOfRows); + blockDataEnsureCapacity(pRes, pBlock->info.rows); // data from mnode - pRes->info.rows = numOfRows; + pRes->info.rows = pBlock->info.rows; relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); blockDataDestroy(pBlock); } // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. blockDataUpdateTsWindow(pRes, 0); - - pLoadInfo->totalRows += numOfRows; - pLoadInfo->totalSize += compLen; - - if (total != NULL) { - *total += numOfRows; - } - return TSDB_CODE_SUCCESS; } @@ -2148,9 +2148,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn while(index++ < pRetrieveRsp->numOfBlocks) { SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); blockDataEnsureCapacity(pb, pRetrieveRsp->numOfRows); - code = - extractDataBlockFromFetchRsp(pb, pLoadInfo, pRetrieveRsp->numOfRows, pStart, - pRetrieveRsp->compLen, pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); + code = extractDataBlockFromFetchRsp(pb, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); goto _error; @@ -2159,7 +2157,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn taosArrayPush(pExchangeInfo->pResultBlockList, &pb); } - pLoadInfo->totalElapsed += (taosGetTimestampUs() - startTs); + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d" @@ -2272,8 +2270,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; char* pStart = pRetrieveRsp->data; - int32_t code = extractDataBlockFromFetchRsp(NULL, pLoadInfo, pRetrieveRsp->numOfRows, pStart, pRetrieveRsp->compLen, - pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); + int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 @@ -2291,7 +2288,9 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { pLoadInfo->totalRows, pLoadInfo->totalSize); } - pOperator->resultInfo.totalRows += pRetrieveRsp->numOfRows; + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); + pDataInfo->totalRows += pRetrieveRsp->numOfRows; + taosMemoryFreeClear(pDataInfo->pRsp); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a2e0b68623..0bd0d68cae 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2303,10 +2303,9 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } char* pStart = pRsp->data; - extractDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pRsp->numOfRows, pRsp->data, pRsp->compLen, - pOperator->exprSupp.numOfExprs, NULL, pInfo->scanCols, &pStart); + extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pOperator->exprSupp.numOfExprs, pInfo->scanCols, &pStart); + updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); - //startTs, // todo log the filter info doFilterResult(pInfo); taosMemoryFree(pRsp); From 200a669542175755a1f6d4bc669f41f2d4760047 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Aug 2022 16:34:11 +0800 Subject: [PATCH 39/65] refactor: do some internal refactor. --- include/common/tmsg.h | 6 ++--- source/client/src/clientImpl.c | 6 ++++- source/common/src/tdatablock.c | 34 +++++++++++++++++-------- source/libs/executor/src/executorimpl.c | 6 ++++- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9d001c9534..5ca6a337a8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -296,13 +296,13 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_IDX_ON ((int8_t)0x2) #define COL_SET_NULL ((int8_t)0x10) #define COL_SET_VAL ((int8_t)0x20) -typedef struct SSchema { +struct SSchema { int8_t type; int8_t flags; col_id_t colId; int32_t bytes; char name[TSDB_COL_NAME_LEN]; -} SSchema; +}; #define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0) #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) @@ -648,7 +648,7 @@ typedef struct { }; bool output; // TODO remove it later - int16_t type; + int8_t type; int32_t bytes; uint8_t precision; uint8_t scale; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8357e2d627..480fd99c81 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1762,6 +1762,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 char* p = (char*)pResultInfo->pData; + // version: + int32_t blockVersion = *(int32_t*)p; + p += sizeof(int32_t); + int32_t dataLen = *(int32_t*)p; p += sizeof(int32_t); @@ -1782,7 +1786,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 // check fields for (int32_t i = 0; i < numOfCols; ++i) { int16_t type = *(int16_t*)p; - p += sizeof(int16_t); + p += sizeof(int8_t); int32_t bytes = *(int32_t*)p; p += sizeof(int32_t); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index bcbdf8d786..45bbe96923 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -676,9 +676,9 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { * @return */ size_t blockDataGetSerialMetaSize(uint32_t numOfCols) { - // | total length | total rows | total columns | has column seg| block group id | column schema | each column length | - return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + - numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | + return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + + numOfCols * (sizeof(int8_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); } double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { @@ -1582,7 +1582,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { for (int32_t i = 0; i < sz; i++) { SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); tlen += taosEncodeFixedI16(buf, pColData->info.colId); - tlen += taosEncodeFixedI16(buf, pColData->info.type); + tlen += taosEncodeFixedI8(buf, pColData->info.type); tlen += taosEncodeFixedI32(buf, pColData->info.bytes); tlen += taosEncodeFixedBool(buf, pColData->hasNull); @@ -1614,7 +1614,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { for (int32_t i = 0; i < sz; i++) { SColumnInfoData data = {0}; buf = taosDecodeFixedI16(buf, &data.info.colId); - buf = taosDecodeFixedI16(buf, &data.info.type); + buf = taosDecodeFixedI8(buf, &data.info.type); buf = taosDecodeFixedI32(buf, &data.info.bytes); buf = taosDecodeFixedBool(buf, &data.hasNull); @@ -2074,6 +2074,10 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { // todo extract method + int32_t* version = (int32_t*)data; + *version = 1; + data += sizeof(int32_t); + int32_t* actualLen = (int32_t*)data; data += sizeof(int32_t); @@ -2085,8 +2089,11 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ *cols = numOfCols; data += sizeof(int32_t); - int32_t* hasColumnSegment = (int32_t*)data; - *hasColumnSegment = 1; + // flag segment. + // the inital bit is for column info + int32_t* flagSegment = (int32_t*)data; + *flagSegment = (1<<31); + data += sizeof(int32_t); uint64_t* groupId = (uint64_t*)data; @@ -2095,8 +2102,8 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - *((int16_t*)data) = pColInfoData->info.type; - data += sizeof(int16_t); + *((int8_t*)data) = pColInfoData->info.type; + data += sizeof(int8_t); *((int32_t*)data) = pColInfoData->info.bytes; data += sizeof(int32_t); @@ -2145,6 +2152,10 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { const char* pStart = pData; + int32_t version = *(int32_t*) pStart; + pStart += sizeof(int32_t); + ASSERT(version == 1); + // total length sizeof(int32_t) int32_t dataLen = *(int32_t*)pStart; pStart += sizeof(int32_t); @@ -2158,7 +2169,8 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(int32_t); // has column info segment - int32_t hasColumnInfo = *(int32_t*)pStart; + int32_t flagSeg = *(int32_t*)pStart; + int32_t hasColumnInfo = (flagSeg >> 31); pStart += sizeof(int32_t); // group id sizeof(uint64_t) @@ -2173,7 +2185,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData->info.type = *(int16_t*)pStart; - pStart += sizeof(int16_t); + pStart += sizeof(int8_t); pColInfoData->info.bytes = *(int32_t*)pStart; pStart += sizeof(int32_t); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 752b93bcea..b7e099d0b1 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4071,7 +4071,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else { ASSERT(0); } - pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; + + if (pOperator != NULL) { + pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; + } + return pOperator; } From 0aef734a20c412f3c6ff00a526dffc3ccebb020b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Aug 2022 17:56:26 +0800 Subject: [PATCH 40/65] fix(query): fix syntax error. --- source/libs/executor/src/executorimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 844ee4cc6e..5f5c035523 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1517,7 +1517,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); - doUpdateNumOfRows(pRow, numOfExprs, rowEntryOffset); + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); // no results, continue to check the next one if (pRow->numOfRows == 0) { From 7e1917f105539a5e89563c01a72dea893e5d5d1b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 09:55:03 +0800 Subject: [PATCH 41/65] enh: refine the data requirements of the partition physical plan --- source/libs/function/src/builtins.c | 12 ++++++------ source/libs/planner/src/planOptimizer.c | 4 ++-- source/libs/planner/src/planUtil.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 13ebcd4518..9102137c31 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2452,7 +2452,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "histogram", .type = FUNCTION_TYPE_HISTOGRAM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateHistogram, .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, @@ -2467,7 +2467,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_histogram_partial", .type = FUNCTION_TYPE_HISTOGRAM_PARTIAL, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateHistogramPartial, .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, @@ -2479,7 +2479,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_histogram_merge", .type = FUNCTION_TYPE_HISTOGRAM_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateHistogramMerge, .getEnvFunc = getHistogramFuncEnv, .initFunc = functionSetup, @@ -2491,7 +2491,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "hyperloglog", .type = FUNCTION_TYPE_HYPERLOGLOG, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateHLL, .getEnvFunc = getHLLFuncEnv, .initFunc = functionSetup, @@ -2505,7 +2505,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "_hyperloglog_partial", - .type = FUNCTION_TYPE_HYPERLOGLOG_PARTIAL | FUNC_MGT_TIMELINE_FUNC, + .type = FUNCTION_TYPE_HYPERLOGLOG_PARTIAL, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateHLLPartial, .getEnvFunc = getHLLFuncEnv, @@ -2517,7 +2517,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "_hyperloglog_merge", - .type = FUNCTION_TYPE_HYPERLOGLOG_MERGE | FUNC_MGT_TIMELINE_FUNC, + .type = FUNCTION_TYPE_HYPERLOGLOG_MERGE, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateHLLMerge, .getEnvFunc = getHLLFuncEnv, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 86a4441cff..0e83051538 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1711,7 +1711,7 @@ static bool eliminateProjOptCanChildConditionUseChildTargets(SLogicNode* pChild, if (!cxt.canUse) return false; } if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pChild) && NULL != ((SJoinLogicNode*)pChild)->pOnConditions) { - SJoinLogicNode* pJoinLogicNode = (SJoinLogicNode*)pChild; + SJoinLogicNode* pJoinLogicNode = (SJoinLogicNode*)pChild; CheckNewChildTargetsCxt cxt = {.pNewChildTargets = pNewChildTargets, .canUse = false}; nodesWalkExpr(pJoinLogicNode->pOnConditions, eliminateProjOptCanUseNewChildTargetsImpl, &cxt); if (!cxt.canUse) return false; @@ -1768,7 +1768,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pProjectNode->node.pChildren); nodesDestroyNode((SNode*)pProjectNode); - //if pChild is a project logic node, remove its projection which is not reference by its target. + // if pChild is a project logic node, remove its projection which is not reference by its target. alignProjectionWithTarget(pChild); } pCxt->optimized = true; diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index d7310892de..ac1589bec0 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -232,7 +232,8 @@ static int32_t adjustPartitionDataRequirement(SPartitionLogicNode* pPart, EDataO return TSDB_CODE_PLAN_INTERNAL_ERROR; } pPart->node.resultDataOrder = requirement; - pPart->node.requireDataOrder = requirement; + pPart->node.requireDataOrder = + (requirement >= DATA_ORDER_LEVEL_IN_BLOCK ? DATA_ORDER_LEVEL_GLOBAL : DATA_ORDER_LEVEL_NONE); return TSDB_CODE_SUCCESS; } From 30a64bdcd9315dedbd69c0680ec38e9a5b508273 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 10:16:00 +0800 Subject: [PATCH 42/65] fix(query): set correct schema. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 139 ++++++++++++------------- 1 file changed, 68 insertions(+), 71 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0749d7b133..e1e236bf11 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -14,6 +14,7 @@ */ #include "tsdb.h" +#include "osDef.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) typedef enum { @@ -129,7 +130,8 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; - STSchema* pSchema; + STSchema* pSchema;// the newest version schema + STSchema* pMemSchema;// the previous schema for in-memory data, to avoid load schema too many times SDataFReader* pFileReader; SVersionRange verRange; @@ -145,11 +147,10 @@ static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanI SRowMerger* pMerger); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); -static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow); +static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); static int32_t doAppendRowFromBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); -static void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); static void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, @@ -1230,6 +1231,31 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB return false; } +static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { + // always set the newest schema version in pReader->pSchema + if (pReader->pSchema == NULL) { + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1); + } + + if (sversion == pReader->pSchema->version) { + return pReader->pSchema; + } + + if (pReader->pMemSchema == NULL) { + int32_t code = + metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); + return pReader->pMemSchema; + } + + if (pReader->pMemSchema->version == sversion) { + return pReader->pMemSchema; + } + + taosMemoryFree(pReader->pMemSchema); + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); + return pReader->pMemSchema; +} + static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key) { SRowMerger merge = {0}; @@ -1241,6 +1267,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SArray* pDelList = pBlockScanInfo->delSkyline; bool freeTSRow = false; + uint64_t uid = pBlockScanInfo->uid; // ascending order traverse if (ASCENDING_TRAVERSE(pReader->order)) { @@ -1276,9 +1303,9 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerGetRow(&merge, &pTSRow); } } else { // descending order: mem rows -----> imem rows ------> file block - updateSchema(pRow, pBlockScanInfo->uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pReader->pSchema); + tRowMergerInit(&merge, pRow, pSchema); doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, &fRow); @@ -1289,7 +1316,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } tRowMergerClear(&merge); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; @@ -1333,7 +1360,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* } tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); return TSDB_CODE_SUCCESS; } else { // key > ik.ts || key > k.ts ASSERT(key != ik.ts); @@ -1342,7 +1369,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [4] ik.ts < k.ts <= key if (ik.ts < k.ts) { doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -1353,7 +1380,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [6] k.ts < ik.ts <= key if (k.ts < ik.ts) { doMergeMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -1365,7 +1392,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* ASSERT(key > ik.ts && key > k.ts); doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } @@ -1373,9 +1400,9 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { // descending order scan // [1/2] k.ts >= ik.ts && k.ts >= key if (k.ts >= ik.ts && k.ts >= key) { - updateSchema(pRow, uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pReader->pSchema); + tRowMergerInit(&merge, pRow, pSchema); doMergeRowsInBuf(&pBlockScanInfo->iter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader); if (ik.ts == k.ts) { @@ -1390,7 +1417,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* } tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); return TSDB_CODE_SUCCESS; } else { ASSERT(ik.ts != k.ts); // this case has been included in the previous if branch @@ -1399,7 +1426,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [4] ik.ts > key >= k.ts if (ik.ts > key) { doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -1414,7 +1441,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } @@ -1427,7 +1454,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; @@ -1495,7 +1522,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); taosMemoryFree(pTSRow); tRowMergerClear(&merge); @@ -2146,6 +2173,7 @@ TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pRea } } + int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader) { while (1) { @@ -2166,22 +2194,8 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe break; } - int32_t sversion = TSDBROW_SVERSION(pRow); - STSchema* pTSchema = NULL; - if (pReader->pSchema == NULL || sversion != pReader->pSchema->version) { - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pTSchema); - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } - } else { - pTSchema = pReader->pSchema; - } - + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); tRowMergerAdd(pMerger, pRow, pTSchema); - - if (pTSchema != pReader->pSchema) { - taosMemoryFree(pTSchema); - } } return TSDB_CODE_SUCCESS; @@ -2289,21 +2303,13 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc return TSDB_CODE_SUCCESS; } -void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader) { - int32_t sversion = TSDBROW_SVERSION(pRow); - - if (pReader->pSchema == NULL) { - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pSchema); - } else if (pReader->pSchema->version != sversion) { - taosMemoryFreeClear(pReader->pSchema); - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pSchema); - } -} - void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow) { + + TSDBROW* pNextRow = NULL; + TSDBROW current = *pRow; + { // if the timestamp of the next valid row has a different ts, return current row directly - TSDBROW current = *pRow; pIter->hasVal = tsdbTbDataIterNext(pIter->iter); if (!pIter->hasVal) { @@ -2311,14 +2317,14 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe *freeTSRow = false; return; } else { // has next point in mem/imem - TSDBROW* pNextRow = getValidRow(pIter, pDelList, pReader); + pNextRow = getValidRow(pIter, pDelList, pReader); if (pNextRow == NULL) { *pTSRow = current.pTSRow; *freeTSRow = false; return; } - if (TSDBROW_KEY(¤t).ts != TSDBROW_KEY(pNextRow).ts) { + if (current.pTSRow->ts != pNextRow->pTSRow->ts) { *pTSRow = current.pTSRow; *freeTSRow = false; return; @@ -2327,30 +2333,20 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe } SRowMerger merge = {0}; - TSDBKEY k = TSDBROW_KEY(pRow); // get the correct schema for data in memory - int32_t sversion = TSDBROW_SVERSION(pRow); - STSchema* pTSchema = NULL; - if (pReader->pSchema == NULL || sversion != pReader->pSchema->version) { - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pTSchema); - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } - } else { - pTSchema = pReader->pSchema; - } + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); - tRowMergerInit2(&merge, pReader->pSchema, pRow, pTSchema); - doMergeRowsInBuf(pIter, uid, k.ts, pDelList, &merge, pReader); + tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + + STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + tRowMergerAdd(&merge, pNextRow, pTSchema1); + + doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); tRowMergerGetRow(&merge, pTSRow); tRowMergerClear(&merge); *freeTSRow = true; - - if (sversion != pReader->pSchema->version) { - taosMemoryFree(pTSchema); - } } void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, @@ -2361,17 +2357,17 @@ void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlo TSDBKEY ik = TSDBROW_KEY(piRow); if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem - updateSchema(piRow, pBlockScanInfo->uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pReader->pSchema); + tRowMergerInit(&merge, piRow, pSchema); doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, pRow); doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } else { - updateSchema(pRow, pBlockScanInfo->uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pReader->pSchema); + tRowMergerInit(&merge, pRow, pSchema); doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, piRow); @@ -2432,12 +2428,12 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow) { +int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid) { int32_t numOfRows = pBlock->info.rows; int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - STSchema* pSchema = pReader->pSchema; + STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); SColVal colVal = {0}; int32_t i = 0, j = 0; @@ -2453,7 +2449,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* col_id_t colId = pColInfoData->info.colId; if (colId == pSchema->columns[j].colId) { - tTSRowGetVal(pTSRow, pReader->pSchema, j, &colVal); + tTSRowGetVal(pTSRow, pSchema, j, &colVal); doCopyColVal(pColInfoData, numOfRows, i, &colVal, pSupInfo); i += 1; j += 1; @@ -2529,7 +2525,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e break; } - doAppendRowFromTSRow(pBlock, pReader, pTSRow); + doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo->uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -2755,6 +2751,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); + taosMemoryFree(pReader->pMemSchema); taosMemoryFreeClear(pReader); } From f13215a9016adc902b97a7ff0b9f2e05af9623f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 10:17:48 +0800 Subject: [PATCH 43/65] test:fix invalid test cases. --- tests/script/tsim/parser/insert_multiTbl.sim | 2 +- tests/script/tsim/parser/last_cache_query.sim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/tsim/parser/insert_multiTbl.sim b/tests/script/tsim/parser/insert_multiTbl.sim index 78d3352378..fb6da1cfba 100644 --- a/tests/script/tsim/parser/insert_multiTbl.sim +++ b/tests/script/tsim/parser/insert_multiTbl.sim @@ -49,7 +49,7 @@ if $rows != 12 then return -1 endi if $data02 != NULL then - print "expect: NULL, act: $data03" + print "expect: NULL, act: $data02" return -1 endi if $data03 != @binary00@ then diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim index 7bafe82f5d..ebbb784da5 100644 --- a/tests/script/tsim/parser/last_cache_query.sim +++ b/tests/script/tsim/parser/last_cache_query.sim @@ -304,8 +304,8 @@ endi if $data21 != 24 then return -1 endi -if $data22 != 11.000000000 then - print $data02 +if $data22 != 11.000000000 then + print expect 11.000000000 actual: $data22 return -1 endi if $data23 != 25 then From 29c84833d3d7916d1ef83418e7fb1931d7b537e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 10:34:01 +0800 Subject: [PATCH 44/65] fix(query):fix invalid free. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index e1e236bf11..714af80e83 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1279,6 +1279,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } else if (k.ts < key) { // k.ts < key doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader, &freeTSRow); @@ -1290,6 +1291,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } else { // descending order scan if (key < k.ts) { @@ -1301,6 +1303,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } else { // descending order: mem rows -----> imem rows ------> file block STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); @@ -1312,13 +1315,17 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } tRowMergerClear(&merge); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - taosMemoryFree(pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } + return TSDB_CODE_SUCCESS; } From 43a67f6da9b8c990a03cbb93be5f20fca7f15487 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 10:57:44 +0800 Subject: [PATCH 45/65] fix: insert error of unbound columns --- source/libs/parser/src/parInsert.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 6e47e26a7a..4756f65bf5 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -18,10 +18,10 @@ #include "parInt.h" #include "parToken.h" #include "parUtil.h" +#include "query.h" #include "tglobal.h" #include "ttime.h" #include "ttypes.h" -#include "query.h" #define NEXT_TOKEN(pSql, sToken) \ do { \ @@ -1189,12 +1189,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, if (!isParseBindParam) { // set the null value for the columns that do not assign values if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(schema[i].type), true, schema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } *gotRow = true; @@ -1267,7 +1262,8 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) { - return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than INT32_MAX"); + return buildInvalidOperationMsg(&pCxt->msg, + "too many rows in sql, total number of rows should be less than INT32_MAX"); } dataBuf->numOfTables = 1; @@ -1339,7 +1335,8 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STa SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) { - return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than INT32_MAX"); + return buildInvalidOperationMsg(&pCxt->msg, + "too many rows in sql, total number of rows should be less than INT32_MAX"); } dataBuf->numOfTables = 1; @@ -2060,7 +2057,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu } } } - if(rowEnd) { + if (rowEnd) { tdSRowEnd(pBuilder); } #ifdef TD_DEBUG_PRINT_ROW @@ -2077,7 +2074,8 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { - return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than INT32_MAX"); + return buildInvalidOperationMsg(&pBuf, + "too many rows in sql, total number of rows should be less than INT32_MAX"); } } From a2a1da06dd056b58d040fb83957a239377b9520b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 13:45:44 +0800 Subject: [PATCH 46/65] other:merge 3.0 --- contrib/CMakeLists.txt | 17 +++++++ include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 57 ++++++++++++++--------- source/dnode/vnode/src/tq/tqExec.c | 13 +++++- source/libs/executor/src/dataDispatcher.c | 8 ++-- source/libs/executor/src/executor.c | 33 +++++++++---- source/libs/qworker/src/qworker.c | 29 ++++++------ source/libs/stream/src/streamExec.c | 23 ++++++--- 8 files changed, 124 insertions(+), 58 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b4e8825431..de7b75a245 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -27,6 +27,10 @@ else () cat("${TD_SUPPORT_DIR}/taosadapter_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() +if(TD_LINUX_64 AND JEMALLOC_ENABLED) + cat("${TD_SUPPORT_DIR}/jemalloc_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + # pthread if(${BUILD_PTHREAD}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -392,6 +396,19 @@ if(${BUILD_WITH_SQLITE}) endif(NOT TD_WINDOWS) endif(${BUILD_WITH_SQLITE}) +# jemalloc +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + include(ExternalProject) + ExternalProject_Add(jemalloc + PREFIX "jemalloc" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ + BUILD_COMMAND ${MAKE} + ) + INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include) +ENDIF () + # addr2line if(${BUILD_ADDR2LINE}) if(NOT ${TD_WINDOWS}) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 7bc2ede4fc..a3cc7aaa8a 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -123,7 +123,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table * @param handle * @return */ -int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds); +int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds); /** * kill the ongoing query and free the query handle and corresponding resources automatically diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 662558529d..ef9a64e946 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -604,28 +604,44 @@ _end: static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, int8_t blkType) { - while (1) { - SSDataBlock *output = NULL; - uint64_t ts; + SArray *pResList = taosArrayInit(1, POINTER_BYTES); + if (pResList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } - int32_t code = qExecTask(taskInfo, &output, &ts); + while (1) { + uint64_t ts; + int32_t code = qExecTask(taskInfo, pResList, &ts); if (code < 0) { smaError("vgId:%d, qExecTask for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr(code)); goto _err; } - if (output) { -#if 0 + if (taosArrayGetSize(pResList) == 0) { + if (terrno == 0) { + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); + } else { + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, terrstr()); + goto _err; + } + + break; + } + + for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { + SSDataBlock *output = taosArrayGetP(pResList, i); + +#if 1 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, pItem->level); - SArray *pResult = taosArrayInit(1, sizeof(SSDataBlock)); - taosArrayPush(pResult, output); - blockDebugShowDataBlocks(pResult, flag); - taosArrayDestroy(pResult); +// blockDebugShowDataBlocks(output, flag); +// taosArrayDestroy(pResult); #endif - STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); + STsdb * sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); SSubmitReq *pReq = NULL; + // TODO: the schema update should be handled later(TD-17965) if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, SMA_VID(pSma), suid) < 0) { smaError("vgId:%d, build submit req for rsma stable %" PRIi64 " level %" PRIi8 " failed since %s", @@ -644,17 +660,14 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm SMA_VID(pSma), suid, pItem->level, output->info.version); taosMemoryFreeClear(pReq); - } else if (terrno == 0) { - smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); - break; - } else { - smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, terrstr()); - goto _err; } } + taosArrayDestroy(pResList); return TSDB_CODE_SUCCESS; + _err: + taosArrayDestroy(pResList); return TSDB_CODE_FAILED; } @@ -1407,7 +1420,7 @@ int32_t tdRSmaFetchSend(SSma *pSma, SRSmaInfo *pInfo, int8_t level) { tEncoderClear(&encoder); } tEncoderClear(&encoder); - + ((SMsgHead *)pBuf)->vgId = SMA_VID(pSma); ((SMsgHead *)pBuf)->contLen = contLen + sizeof(SMsgHead); @@ -1434,10 +1447,10 @@ _err: /** * @brief fetch rsma data of level 2/3 and submit - * - * @param pSma - * @param pMsg - * @return int32_t + * + * @param pSma + * @param pMsg + * @return int32_t */ int32_t smaProcessFetch(SSma *pSma, void *pMsg) { SRpcMsg *pRpcMsg = (SRpcMsg *)pMsg; diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 435bbb77b8..09a5f1a8c6 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -80,14 +80,22 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa } int32_t rowCnt = 0; + SArray* pResList = taosArrayInit(4, POINTER_BYTES); + while (1) { + taosArrayClear(pResList); + SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; tqDebug("task start to execute"); - if (qExecTask(task, &pDataBlock, &ts) < 0) { + if (qExecTask(task, pResList, &ts) < 0) { ASSERT(0); } - tqDebug("task execute end, get %p", pDataBlock); + + if (taosArrayGetSize(pResList) > 0) { + pDataBlock = taosArrayGet(pResList, 0); + tqDebug("task execute end, get %p", pDataBlock); + } if (pDataBlock != NULL) { if (pRsp->withTbName) { @@ -143,6 +151,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa break; } + taosArrayDestroy(pResList); return 0; } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 3a37b5e760..977adcaa5e 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -69,10 +69,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { // clang-format off // data format: -// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | total length | numOfRows | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) -// | |sizeof(int32) | sizeof(int32)| sizeof(uint64_t) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | -// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// +----------------+------------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// |SDataCacheEntry | version | total length | numOfRows | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) +// | | sizeof(int32_t) |sizeof(int32) | sizeof(int32)| sizeof(uint64_t) | (sizeof(int8_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | +// +----------------+------------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header // clang-format on diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 897ddb59ec..5bbf5928da 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -422,11 +422,17 @@ int waitMoment(SQInfo* pQInfo) { } #endif -int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { +static void freeBlock(void* param) { + SSDataBlock* pBlock = *(SSDataBlock**) param; + blockDataDestroy(pBlock); +} + +int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); - *pRes = NULL; + taosArrayClearEx(pResList, freeBlock); + int64_t curOwner = 0; if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); @@ -457,23 +463,34 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); + int32_t current = 0; + SSDataBlock* pRes = NULL; + int64_t st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); + while((pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot)) != NULL) { + SSDataBlock* p = createOneDataBlock(pRes, true); + current += p->info.rows; + ASSERT(p->info.rows > 0); + taosArrayPush(pResList, &p); + + if (current >= 4096) { + break; + } + } + uint64_t el = (taosGetTimestampUs() - st); pTaskInfo->cost.elapsedTime += el; - if (NULL == *pRes) { + if (NULL == pRes) { *useconds = pTaskInfo->cost.elapsedTime; } cleanUpUdfs(); - - int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; - qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", - GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0); + qDebug("%s task suspended, %d rows in %d blocks returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", + GET_TASKID(pTaskInfo), current, (int32_t) taosArrayGetSize(pResList), total, 0, el / 1000.0); atomic_store_64(&pTaskInfo->owner, 0); return pTaskInfo->code; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 7c5cfda81f..85a1b0444c 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -75,22 +75,20 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { int32_t code = 0; bool qcontinue = true; - SSDataBlock *pRes = NULL; uint64_t useconds = 0; int32_t i = 0; int32_t execNum = 0; qTaskInfo_t taskHandle = ctx->taskHandle; DataSinkHandle sinkHandle = ctx->sinkHandle; + SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (true) { QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); - pRes = NULL; - // if *taskHandle is NULL, it's killed right now if (taskHandle) { qwDbgSimulateSleep(); - code = qExecTask(taskHandle, &pRes, &useconds); + code = qExecTask(taskHandle, pResList, &useconds); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -103,9 +101,8 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { ++execNum; - if (NULL == pRes) { + if (taosArrayGetSize(pResList) == 0) { QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); - dsEndPut(sinkHandle, useconds); QW_ERR_RET(qwHandleTaskComplete(QW_FPARAMS(), ctx)); @@ -117,19 +114,20 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { break; } - int32_t rows = pRes->info.rows; + for(int32_t j = 0; j < taosArrayGetSize(pResList); ++j) { + SSDataBlock *pRes = taosArrayGetP(pResList, j); + ASSERT(pRes->info.rows > 0); - ASSERT(pRes->info.rows > 0); + SInputData inputData = {.pData = pRes}; + code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); + if (code) { + QW_TASK_ELOG("dsPutDataBlock failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); + } - SInputData inputData = {.pData = pRes}; - code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); - if (code) { - QW_TASK_ELOG("dsPutDataBlock failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); + QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); } - QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", rows, qcontinue); - if (!qcontinue) { if (queryStop) { *queryStop = true; @@ -151,6 +149,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { } } + taosArrayDestroy(pResList); QW_RET(code); } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 7512f792c1..b92f658de0 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -43,13 +43,17 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* } // exec + SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SSDataBlock* output = NULL; uint64_t ts = 0; - if (qExecTask(exec, &output, &ts) < 0) { + + taosArrayClear(pResList); + if (qExecTask(exec, pResList, &ts) < 0) { ASSERT(false); } - if (output == NULL) { + + if (taosArrayGetSize(pResList) == 0) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SSDataBlock block = {0}; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data; @@ -65,6 +69,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* break; } + output = taosArrayGetP(pResList, 0); if (output->info.type == STREAM_RETRIEVE) { if (streamBroadcastToChildren(pTask, output) < 0) { // TODO @@ -79,6 +84,8 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); } + + taosArrayDestroy(pResList); return 0; } @@ -98,6 +105,7 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { void* exec = pTask->exec.executor; + SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { @@ -107,14 +115,17 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { int32_t batchCnt = 0; while (1) { - SSDataBlock* output = NULL; - uint64_t ts = 0; - if (qExecTask(exec, &output, &ts) < 0) { + uint64_t ts = 0; + taosArrayClear(pResList); + if (qExecTask(exec, pResList, &ts) < 0) { ASSERT(0); } - if (output == NULL) break; + + if (taosArrayGetSize(pResList) == 0) break; SSDataBlock block = {0}; + SSDataBlock* output = taosArrayGetP(pResList, 0); + assignOneDataBlock(&block, output); block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); From b9c6e0cdf9db43c6b56b3cf7ec7757ea2ee79843 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 13:55:51 +0800 Subject: [PATCH 47/65] refactor: stream invoke previous APIs. --- include/libs/executor/executor.h | 3 +- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/tq/tqExec.c | 13 +----- source/libs/executor/src/executor.c | 59 +++++++++++++++++++++++++- source/libs/qworker/src/qworker.c | 2 +- source/libs/stream/src/streamExec.c | 23 +++------- 6 files changed, 70 insertions(+), 32 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index a3cc7aaa8a..a64815f14f 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -123,7 +123,8 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table * @param handle * @return */ -int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds); +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds); +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds); /** * kill the ongoing query and free the query handle and corresponding resources automatically diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ef9a64e946..6b513f0242 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -612,7 +612,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm while (1) { uint64_t ts; - int32_t code = qExecTask(taskInfo, pResList, &ts); + int32_t code = qExecTaskOpt(taskInfo, pResList, &ts); if (code < 0) { smaError("vgId:%d, qExecTask for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr(code)); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 09a5f1a8c6..435bbb77b8 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -80,22 +80,14 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa } int32_t rowCnt = 0; - SArray* pResList = taosArrayInit(4, POINTER_BYTES); - while (1) { - taosArrayClear(pResList); - SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; tqDebug("task start to execute"); - if (qExecTask(task, pResList, &ts) < 0) { + if (qExecTask(task, &pDataBlock, &ts) < 0) { ASSERT(0); } - - if (taosArrayGetSize(pResList) > 0) { - pDataBlock = taosArrayGet(pResList, 0); - tqDebug("task execute end, get %p", pDataBlock); - } + tqDebug("task execute end, get %p", pDataBlock); if (pDataBlock != NULL) { if (pRsp->withTbName) { @@ -151,7 +143,6 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa break; } - taosArrayDestroy(pResList); return 0; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 5bbf5928da..328a65bec4 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -427,7 +427,7 @@ static void freeBlock(void* param) { blockDataDestroy(pBlock); } -int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); @@ -496,6 +496,63 @@ int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { return pTaskInfo->code; } +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + int64_t threadId = taosGetSelfPthreadId(); + + *pRes = NULL; + int64_t curOwner = 0; + if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { + qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); + pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; + return pTaskInfo->code; + } + + if (pTaskInfo->cost.start == 0) { + pTaskInfo->cost.start = taosGetTimestampMs(); + } + + if (isTaskKilled(pTaskInfo)) { + atomic_store_64(&pTaskInfo->owner, 0); + qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); + return TSDB_CODE_SUCCESS; + } + + // error occurs, record the error code and return to client + int32_t ret = setjmp(pTaskInfo->env); + if (ret != TSDB_CODE_SUCCESS) { + pTaskInfo->code = ret; + cleanUpUdfs(); + qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); + atomic_store_64(&pTaskInfo->owner, 0); + + return pTaskInfo->code; + } + + qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); + + int64_t st = taosGetTimestampUs(); + + *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); + uint64_t el = (taosGetTimestampUs() - st); + + pTaskInfo->cost.elapsedTime += el; + if (NULL == *pRes) { + *useconds = pTaskInfo->cost.elapsedTime; + } + + cleanUpUdfs(); + + int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; + uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; + + qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", + GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0); + + atomic_store_64(&pTaskInfo->owner, 0); + return pTaskInfo->code; +} + int32_t qKillTask(qTaskInfo_t qinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; if (pTaskInfo == NULL) { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 85a1b0444c..e06b752862 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -88,7 +88,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { // if *taskHandle is NULL, it's killed right now if (taskHandle) { qwDbgSimulateSleep(); - code = qExecTask(taskHandle, pResList, &useconds); + code = qExecTaskOpt(taskHandle, pResList, &useconds); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index b92f658de0..7512f792c1 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -43,17 +43,13 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* } // exec - SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SSDataBlock* output = NULL; uint64_t ts = 0; - - taosArrayClear(pResList); - if (qExecTask(exec, pResList, &ts) < 0) { + if (qExecTask(exec, &output, &ts) < 0) { ASSERT(false); } - - if (taosArrayGetSize(pResList) == 0) { + if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SSDataBlock block = {0}; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data; @@ -69,7 +65,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* break; } - output = taosArrayGetP(pResList, 0); if (output->info.type == STREAM_RETRIEVE) { if (streamBroadcastToChildren(pTask, output) < 0) { // TODO @@ -84,8 +79,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); } - - taosArrayDestroy(pResList); return 0; } @@ -105,7 +98,6 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { void* exec = pTask->exec.executor; - SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { @@ -115,17 +107,14 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { int32_t batchCnt = 0; while (1) { - uint64_t ts = 0; - taosArrayClear(pResList); - if (qExecTask(exec, pResList, &ts) < 0) { + SSDataBlock* output = NULL; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { ASSERT(0); } - - if (taosArrayGetSize(pResList) == 0) break; + if (output == NULL) break; SSDataBlock block = {0}; - SSDataBlock* output = taosArrayGetP(pResList, 0); - assignOneDataBlock(&block, output); block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); From 9b1201a61ae00d5ca919d11dbda4aabf74b36258 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 15:18:21 +0800 Subject: [PATCH 48/65] fix: slimit error --- source/libs/planner/src/planLogicCreater.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 12646c773c..875ec21b24 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -469,7 +469,8 @@ static int32_t createGroupKeysFromPartKeys(SNodeList* pPartKeys, SNodeList** pOu } static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { - return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; + return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP + : GROUP_ACTION_NONE; } static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { From 9e498a7859ea354e82cc13caa4035e85d3ce0003 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 9 Aug 2022 15:40:39 +0800 Subject: [PATCH 49/65] fix: set none for unbound cols --- source/libs/parser/src/parInsert.c | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 4756f65bf5..f2d0ede486 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1966,12 +1966,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in } // set the null value for the columns that do not assign values if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } tdSRowEnd(pBuilder); #ifdef TD_DEBUG_PRINT_ROW @@ -2050,12 +2045,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu // set the null value for the columns that do not assign values if (rowEnd && (spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } if (rowEnd) { tdSRowEnd(pBuilder); @@ -2432,12 +2422,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols // set the null value for the columns that do not assign values if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } tdSRowEnd(pBuilder); From 41d2c922311542eb0a393da065df3d0c52229f50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 16:54:05 +0800 Subject: [PATCH 50/65] fix(query): copy null value. --- source/common/src/tdatablock.c | 4 ---- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9a1d761e0d..dba30bb876 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1329,10 +1329,6 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); - if (pSrc->pData == NULL) { - continue; - } - colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 714af80e83..37861a0e0a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -182,6 +182,7 @@ static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes); + tsdbInfo("-------------------%d\n", pCol->info.bytes); } } @@ -674,6 +675,7 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ colDataAppendNULL(pColInfoData, rowIndex); } else { varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); + ASSERT(pColVal->value.nData <= pColInfoData->info.bytes); memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); colDataAppend(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); } From 8b86d7342d89be79b32dd70a076e14cdad6fbdd5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 17:21:31 +0800 Subject: [PATCH 51/65] fix(query): check the sma load status, and return if succeed. --- source/libs/executor/src/scanoperator.c | 10 +++++++--- tests/script/tsim/testsuit.sim | 14 +++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b2fb80b68f..e401a1bd96 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -312,9 +312,13 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; - loadSMA = true; // mark the operator of load sma; + loadSMA = true; // mark the operation of load sma; bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); - if (!success) { // failed to load the block sma data, data block statistics does not exist, load data block instead + if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead + qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + return TSDB_CODE_SUCCESS; + } else { *status = FUNC_DATA_REQUIRED_DATA_LOAD; } } @@ -339,7 +343,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } } - // try to filter datablock according to current results + // try to filter data block according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index e0c46ae5fe..89a4babb0b 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -1,10 +1,10 @@ -run tsim/user/password.sim -run tsim/user/privilege_db.sim -run tsim/user/privilege_sysinfo.sim -run tsim/user/basic.sim -run tsim/table/basic1.sim -run tsim/trans/lossdata1.sim -run tsim/trans/create_db.sim +#run tsim/user/password.sim +#run tsim/user/privilege_db.sim +#run tsim/user/privilege_sysinfo.sim +#run tsim/user/basic.sim +#run tsim/table/basic1.sim +#run tsim/trans/lossdata1.sim +#run tsim/trans/create_db.sim run tsim/stable/alter_metrics.sim run tsim/stable/tag_modify.sim run tsim/stable/alter_comment.sim From bddb3e356d06def42f7430caeef971007f425d80 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 17:55:00 +0800 Subject: [PATCH 52/65] fix(query): expand the sink buffer. --- source/libs/executor/src/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 328a65bec4..497681ded1 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -353,7 +353,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, goto _error; } - SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100}; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 10000, .maxDataBlockNumPerQuery = 5000}; code = dsDataSinkMgtInit(&cfg); if (code != TSDB_CODE_SUCCESS) { goto _error; From d680da33806a3b0e3fbaac48a3dd01cbcfd73721 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 18:35:10 +0800 Subject: [PATCH 53/65] fix(query): fix sma info in query procesing. --- source/libs/executor/src/executor.c | 4 ++++ source/libs/executor/src/scanoperator.c | 3 +++ tests/script/tsim/parser/limit1_tb.sim | 4 ++++ tests/script/tsim/parser/mixed_blocks.sim | 1 + 4 files changed, 12 insertions(+) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 497681ded1..1fc0f5ff51 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -348,6 +348,8 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, taosThreadOnce(&initPoolOnce, initRefPool); atexit(cleanupRefPool); + qDebug("start to create subplan task, TID:0x%"PRIx64 " QID:0x%"PRIx64, taskId, pSubplan->id.queryId); + int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -372,6 +374,8 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } } + qDebug("subplan task create completed, TID:0x%"PRIx64 " QID:0x%"PRIx64, taskId, pSubplan->id.queryId); + _error: // if failed to add ref for all tables in this query, abort current query return code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e401a1bd96..a0f2560e7b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -343,6 +343,9 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } } + // free the sma info, since it should not be involved in later computing process. + taosMemoryFreeClear(pBlock->pBlockAgg); + // try to filter data block according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { diff --git a/tests/script/tsim/parser/limit1_tb.sim b/tests/script/tsim/parser/limit1_tb.sim index 1ef6a62291..36c91fe4c7 100644 --- a/tests/script/tsim/parser/limit1_tb.sim +++ b/tests/script/tsim/parser/limit1_tb.sim @@ -263,6 +263,9 @@ sql select sum(c1), avg(c2), stddev(c3), max(c4), min(c5), count(c6), first(c7), if $rows != 0 then return -1 endi + +print select sum(c1), avg(c2), stddev(c3), max(c4), min(c5), count(c6), first(c7), last(c8), last(c9) from $tb where ts >= $ts0 and ts <= $tsu and c1>1 and c2<9 and c3>2 and c4<8 and c5>4 and c6<6 limit 1 offset 0 + sql select sum(c1), avg(c2), stddev(c3), max(c4), min(c5), count(c6), first(c7), last(c8), last(c9) from $tb where ts >= $ts0 and ts <= $tsu and c1>1 and c2<9 and c3>2 and c4<8 and c5>4 and c6<6 limit 1 offset 0 if $rows != 1 then return -1 @@ -270,6 +273,7 @@ endi $val = $rowNum / 10 $val = $val * 5 if $data00 != $val then + print $data00, expect $val return -1 endi if $data01 != 5.000000000 then diff --git a/tests/script/tsim/parser/mixed_blocks.sim b/tests/script/tsim/parser/mixed_blocks.sim index 76ac7c1c54..80cdfcac99 100644 --- a/tests/script/tsim/parser/mixed_blocks.sim +++ b/tests/script/tsim/parser/mixed_blocks.sim @@ -92,6 +92,7 @@ if $data03 != 319 then return -1 endi if $data04 != 318.000000000 then + print expect 318.000000000, actual: $data04 return -1 endi if $data05 != 1 then From 7b70f54a6e3c30accbd4e3fe50d5500f1b812b4e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 18:47:23 +0800 Subject: [PATCH 54/65] test: add log. --- source/libs/executor/src/scanoperator.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a0f2560e7b..a211542de1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2665,11 +2665,17 @@ _error: int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idStr) { + + int64_t st = taosGetTimestampUs(); + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { return code; } + int64_t st1 = taosGetTimestampUs(); + qDebug("generate queried table list completed, elapsed time:%.2f ms %s", (st1-st)/1000.0, idStr); + if (taosArrayGetSize(pTableListInfo->pTableList) == 0) { qDebug("no table qualified for query, %s" PRIx64, idStr); return TSDB_CODE_SUCCESS; @@ -2681,6 +2687,9 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return code; } + int64_t st2 = taosGetTimestampUs(); + qDebug("generate group id map completed, elapsed time:%.2f ms %s", (st2-st1)/1000.0, idStr); + return TSDB_CODE_SUCCESS; } From 3e626d56d15978b39ff3ea835fe21593719aa97c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 9 Aug 2022 20:32:02 +0800 Subject: [PATCH 55/65] fix: set none if cols not full --- source/libs/executor/src/dataInserter.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 8e355ad3df..26e1f0a3ee 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -213,6 +213,9 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); } } + if(!fullCol) { + rb.hasNone = true; + } tdSRowEnd(&rb); if (ignoreRow) { From 7df0a65e75e427d1a0df06c6b2929e48ad072d75 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 20:57:34 +0800 Subject: [PATCH 56/65] fix(query): set correct meta size in handling json. --- source/client/src/clientImpl.c | 12 ++++++++++-- tests/script/tsim/compute/diff2.sim | 1 - 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7838d87a20..d3079a48d7 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1571,10 +1571,18 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int return TSDB_CODE_SUCCESS; } +static int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { + int32_t cols = *(int32_t*) (p + sizeof(int32_t) * 3); + ASSERT(numOfCols == cols); + + return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t)*3 + sizeof(uint64_t) + numOfCols * (sizeof(int8_t) + sizeof(int32_t)); +} + static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { char* p = (char*)pResultInfo->pData; - int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + // version + length + numOfRows + numOfCol + groupId + flag_segment + column_info + int32_t len = getVersion1BlockMetaSize(p, numOfCols); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; @@ -1642,7 +1650,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* p1 = pResultInfo->convertJson; int32_t totalLen = 0; - int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + int32_t len = getVersion1BlockMetaSize(p, numOfCols); memcpy(p1, p, len); p += len; diff --git a/tests/script/tsim/compute/diff2.sim b/tests/script/tsim/compute/diff2.sim index a09bee991e..5a2c3706ee 100644 --- a/tests/script/tsim/compute/diff2.sim +++ b/tests/script/tsim/compute/diff2.sim @@ -78,7 +78,6 @@ endi sql select diff(c7) from $tb sql_error select diff(c8) from $tb sql_error select diff(c9) from $tb -sql_error select diff(ts) from $tb sql select diff(c1), diff(c2) from $tb sql select 2+diff(c1) from $tb From ae7b2954f4918c5f6e7fe6bf07b91c706fa688b1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 21:02:24 +0800 Subject: [PATCH 57/65] test: remove invalid cases. --- tests/system-test/2-query/diff.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index 76d4891a1c..ceef3f93eb 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -83,9 +83,6 @@ class TDTestCase: tdSql.execute(f"insert into {dbname}.stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - tdSql.error(f"select diff(ts) from {dbname}.stb") - tdSql.error(f"select diff(ts) from {dbname}.stb_1") - # tdSql.error(f"select diff(col7) from {dbname}.stb") tdSql.error(f"select diff(col8) from {dbname}.stb") From 597d360d853c1479c76a224a1daee239a6da5314 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 21:31:22 +0800 Subject: [PATCH 58/65] test: remove invalid cases. --- tests/system-test/2-query/function_diff.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/system-test/2-query/function_diff.py b/tests/system-test/2-query/function_diff.py index 2f463e59a0..492703d3ec 100644 --- a/tests/system-test/2-query/function_diff.py +++ b/tests/system-test/2-query/function_diff.py @@ -256,8 +256,6 @@ class TDTestCase: tdSql.error(self.diff_query_form(col='c%')) # col is spercial char 1 tdSql.error(self.diff_query_form(col='c_')) # col is spercial char 2 tdSql.error(self.diff_query_form(col='c.')) # col is spercial char 3 - tdSql.error(self.diff_query_form(col='c3')) # timestamp col - tdSql.error(self.diff_query_form(col='ts')) # Primary key tdSql.error(self.diff_query_form(col='avg(c1)')) # expr col # tdSql.error(self.diff_query_form(col='c6')) # bool col tdSql.query("select diff(c6) from t1") From abc90650944c782b23851f09fac1c7b4f81e94d0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 13:13:25 +0800 Subject: [PATCH 59/65] fix(query): set correct raw block info. --- source/client/inc/clientInt.h | 6 ++---- source/client/src/clientImpl.c | 2 +- source/client/src/tmq.c | 2 +- source/libs/executor/src/executorimpl.c | 1 - source/libs/executor/src/sortoperator.c | 2 +- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index e7a099cf23..f275ae0885 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -101,10 +101,6 @@ typedef struct SQueryExecMetric { int64_t rsp; // receive response from server, us } SQueryExecMetric; -typedef struct SHeartBeatInfo { - void* pTimer; // timer, used to send request msg to mnode -} SHeartBeatInfo; - struct SAppInstInfo { int64_t numOfConns; SCorEpSet mgmtEp; @@ -256,6 +252,8 @@ SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool valida TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly); void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); +int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols); + static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; return (SReqResultInfo*)&msg->resInfo; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d3079a48d7..5c37e6dc15 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1571,7 +1571,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int return TSDB_CODE_SUCCESS; } -static int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { +int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { int32_t cols = *(int32_t*) (p + sizeof(int32_t) * 3); ASSERT(numOfCols == cols); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index e2619a78b1..8f866bde8e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -2947,7 +2947,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; - char* pStart = pData + sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols) - numOfCols * (sizeof(int8_t) + sizeof(int32_t)); int32_t* colLength = (int32_t*)pStart; pStart += sizeof(int32_t) * numOfCols; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5f5c035523..d3ced810c6 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2157,7 +2157,6 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn char* pStart = pRetrieveRsp->data; while(index++ < pRetrieveRsp->numOfBlocks) { SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); - blockDataEnsureCapacity(pb, pRetrieveRsp->numOfRows); code = extractDataBlockFromFetchRsp(pb, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 16f35b1b0d..4dd5e4ec15 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -658,7 +658,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataDestroy(p); - qDebug("%s get sorted row blocks, rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.rows); + qDebug("%s get sorted block, groupId:%0x"PRIx64" rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } From 99cabaee18cf41178b38108aa9c97d3bd1ec18dd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 14:07:51 +0800 Subject: [PATCH 60/65] fix(query):set correct offset. --- source/client/src/tmq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 8f866bde8e..24997a735c 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -2947,7 +2947,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; - char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols) - numOfCols * (sizeof(int8_t) + sizeof(int32_t)); + char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); int32_t* colLength = (int32_t*)pStart; pStart += sizeof(int32_t) * numOfCols; @@ -2977,6 +2977,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) char* data = pCol[k].pData + pCol[k].offset[j]; tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); } } else { From 6d267810a2471a9edb22d3d51b3cba720353d5d0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 15:22:40 +0800 Subject: [PATCH 61/65] fix(query): fix the invalid length for json data. --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5c37e6dc15..acdb3b68b0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1747,7 +1747,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int pStart1 += colLen1; } - *(int32_t*)(pResultInfo->convertJson) = totalLen; + *(int32_t*)(pResultInfo->convertJson + 4) = totalLen; pResultInfo->pData = pResultInfo->convertJson; return TSDB_CODE_SUCCESS; } From 6dfa2836711a6a2a7b140955a79654c9402c11b9 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 10 Aug 2022 15:35:00 +0800 Subject: [PATCH 62/65] fix: use the pre agg value as double when generate block sma for float column and double column --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d612e9bb10..60f0b18a62 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1520,22 +1520,22 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { break; } case TSDB_DATA_TYPE_FLOAT: { - pColAgg->sum += colVal.value.f; - if (pColAgg->min > colVal.value.f) { - pColAgg->min = colVal.value.f; + *(double*)(&pColAgg->sum) += colVal.value.f; + if (*(double*)(&pColAgg->min) > colVal.value.f) { + *(double*)(&pColAgg->min) = colVal.value.f; } - if (pColAgg->max < colVal.value.f) { - pColAgg->max = colVal.value.f; + if (*(double*)(&pColAgg->max) < colVal.value.f) { + *(double*)(&pColAgg->max) = colVal.value.f; } break; } case TSDB_DATA_TYPE_DOUBLE: { - pColAgg->sum += colVal.value.d; - if (pColAgg->min > colVal.value.d) { - pColAgg->min = colVal.value.d; + *(double*)(&pColAgg->sum) += colVal.value.d; + if (*(double*)(&pColAgg->min) > colVal.value.d) { + *(double*)(&pColAgg->min) = colVal.value.d; } - if (pColAgg->max < colVal.value.d) { - pColAgg->max = colVal.value.d; + if (*(double*)(&pColAgg->max) < colVal.value.d) { + *(double*)(&pColAgg->max) = colVal.value.d; } break; } From 3327ae1556c14e02c1c2cb10672b09a6a3ad5b6f Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Wed, 10 Aug 2022 15:45:14 +0800 Subject: [PATCH 63/65] fix(taosAdapter): adapt to new raw_block --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 7a12413377..ee1e7d3b31 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 766dcc4 + GIT_TAG ed6a160 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 18f6fb6123d8fb9386b821d04bd961cd252210bd Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 10 Aug 2022 16:04:03 +0800 Subject: [PATCH 64/65] enh(tmq): speed up consumer recover --- examples/c/tmq.c | 66 ++-- include/libs/stream/tstream.h | 2 +- include/util/taoserror.h | 1 + source/client/src/tmq.c | 173 +++-------- source/common/src/systable.c | 7 +- source/dnode/mnode/impl/src/mndConsumer.c | 12 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tq/tq.c | 3 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/stream/inc/streamInc.h | 2 +- source/libs/stream/src/streamData.c | 6 +- source/libs/stream/src/streamDispatch.c | 76 +---- source/libs/stream/src/streamExec.c | 34 +- source/libs/wal/src/walWrite.c | 2 + source/os/src/osSemaphore.c | 358 ++++++++++------------ source/util/src/terror.c | 1 + 16 files changed, 281 insertions(+), 466 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 1cdd4c02da..fc34915fe7 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -21,17 +21,17 @@ #include "taos.h" static int running = 1; -static char dbName[64] = "tmqdb"; -static char stbName[64] = "stb"; +static char dbName[64] = "tmqdb"; +static char stbName[64] = "stb"; static char topicName[64] = "topicname"; static int32_t msg_process(TAOS_RES* msg) { - char buf[1024]; + char buf[1024]; int32_t rows = 0; const char* topicName = tmq_get_topic_name(msg); - const char* dbName = tmq_get_db_name(msg); - int32_t vgroupId = tmq_get_vgroup_id(msg); + const char* dbName = tmq_get_db_name(msg); + int32_t vgroupId = tmq_get_vgroup_id(msg); printf("topic: %s\n", topicName); printf("db: %s\n", dbName); @@ -41,14 +41,14 @@ static int32_t msg_process(TAOS_RES* msg) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; - TAOS_FIELD* fields = taos_fetch_fields(msg); + TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); - int32_t* length = taos_fetch_lengths(msg); - int32_t precision = taos_result_precision(msg); - const char* tbName = tmq_get_table_name(msg); - rows++; + int32_t* length = taos_fetch_lengths(msg); + int32_t precision = taos_result_precision(msg); + const char* tbName = tmq_get_table_name(msg); + rows++; taos_print_row(buf, row, fields, numOfFields); - printf("row content from %s: %s\n", (tbName != NULL ? tbName : "null table"), buf); + printf("row content from %s: %s\n", (tbName != NULL ? tbName : "table null"), buf); } return rows; @@ -80,7 +80,8 @@ static int32_t init_env() { // create super table printf("create super table\n"); - pRes = taos_query(pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))"); + pRes = taos_query( + pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))"); if (taos_errno(pRes) != 0) { printf("failed to create super table stb, reason:%s\n", taos_errstr(pRes)); return -1; @@ -166,7 +167,6 @@ int32_t create_topic() { } taos_free_result(pRes); - // pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); pRes = taos_query(pConn, "create topic topicname as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topicname, reason:%s\n", taos_errstr(pRes)); @@ -184,26 +184,28 @@ void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { tmq_t* build_consumer() { tmq_conf_res_t code; - tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_t* conf = tmq_conf_new(); code = tmq_conf_set(conf, "enable.auto.commit", "true"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "group.id", "cgrpName"); if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "client.id", "user defined name"); + if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "td.connect.user", "root"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "td.connect.pass", "taosdata"); if (TMQ_CONF_OK != code) return NULL; - code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); + code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "experimental.snapshot.enable", "true"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "msg.with.table.name", "true"); if (TMQ_CONF_OK != code) return NULL; - - tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); - + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); tmq_conf_destroy(conf); return tmq; @@ -211,7 +213,7 @@ tmq_t* build_consumer() { tmq_list_t* build_topic_list() { tmq_list_t* topicList = tmq_list_new(); - int32_t code = tmq_list_append(topicList, "topicname"); + int32_t code = tmq_list_append(topicList, "topicname"); if (code) { return NULL; } @@ -228,18 +230,18 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topicList) { int32_t totalRows = 0; int32_t msgCnt = 0; - int32_t consumeDelay = 5000; + int32_t timeout = 5000; while (running) { - TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, consumeDelay); + TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, timeout); if (tmqmsg) { msgCnt++; totalRows += msg_process(tmqmsg); taos_free_result(tmqmsg); - } else { - break; - } + /*} else {*/ + /*break;*/ + } } - + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); } @@ -256,32 +258,30 @@ int main(int argc, char* argv[]) { tmq_t* tmq = build_consumer(); if (NULL == tmq) { - fprintf(stderr, "%% build_consumer() fail!\n"); + fprintf(stderr, "%% build_consumer() fail!\n"); return -1; } tmq_list_t* topic_list = build_topic_list(); if (NULL == topic_list) { return -1; - } - + } + basic_consume_loop(tmq, topic_list); code = tmq_unsubscribe(tmq); if (code) { fprintf(stderr, "%% Failed to unsubscribe: %s\n", tmq_err2str(code)); - } - else { + } else { fprintf(stderr, "%% unsubscribe\n"); } code = tmq_consumer_close(tmq); if (code) { fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); - } - else { + } else { fprintf(stderr, "%% Consumer closed\n"); } - + return 0; } diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9355d76dcb..32dc9e1866 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -118,7 +118,7 @@ typedef struct { int64_t sourceVer; int64_t reqId; - SArray* blocks; // SArray + SArray* blocks; // SArray } SStreamDataBlock; typedef struct { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 27fb057b44..145e2f7aad 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -617,6 +617,7 @@ int32_t* taosGetErrno(); //tmq #define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000) +#define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001) #ifdef __cplusplus } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 073f772ee4..c4375c85b4 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -122,6 +122,7 @@ enum { TMQ_CONSUMER_STATUS__INIT = 0, TMQ_CONSUMER_STATUS__READY, TMQ_CONSUMER_STATUS__NO_TOPIC, + TMQ_CONSUMER_STATUS__RECOVER, }; enum { @@ -134,10 +135,8 @@ typedef struct { // statistics int64_t pollCnt; // offset - /*int64_t committedOffset;*/ - /*int64_t currentOffset;*/ - STqOffsetVal committedOffsetNew; - STqOffsetVal currentOffsetNew; + STqOffsetVal committedOffset; + STqOffsetVal currentOffset; // connection info int32_t vgId; int32_t vgStatus; @@ -152,7 +151,6 @@ typedef struct { SArray* vgs; // SArray - int8_t isSchemaAdaptive; SSchemaWrapper schema; } SMqClientTopic; @@ -190,10 +188,9 @@ typedef struct { } SMqPollCbParam; typedef struct { - tmq_t* tmq; - int8_t automatic; - int8_t async; - /*int8_t freeOffsets;*/ + tmq_t* tmq; + int8_t automatic; + int8_t async; int32_t waitingRspNum; int32_t totalRspNum; int32_t rspErr; @@ -418,7 +415,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pOffset->val = pVg->currentOffsetNew; + pOffset->val = pVg->currentOffset; int32_t groupLen = strlen(tmq->groupId); memcpy(pOffset->subKey, tmq->groupId, groupLen); @@ -462,7 +459,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT pVg->vgId, pOffset->val.version); // TODO: put into cb - pVg->committedOffsetNew = pVg->currentOffsetNew; + pVg->committedOffset = pVg->currentOffset; pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; @@ -504,7 +501,6 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm pParamSet->tmq = tmq; pParamSet->automatic = 0; pParamSet->async = async; - /*pParamSet->freeOffsets = 1;*/ pParamSet->userCb = userCb; pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); @@ -518,7 +514,7 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); if (pVg->vgId != vgId) continue; - if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { goto FAIL; } @@ -550,8 +546,8 @@ FAIL: return 0; } -int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, - void* userParam) { +int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, + void* userParam) { int32_t code = -1; if (msg != NULL) { @@ -566,7 +562,6 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ pParamSet->tmq = tmq; pParamSet->automatic = automatic; pParamSet->async = async; - /*pParamSet->freeOffsets = 1;*/ pParamSet->userCb = userCb; pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); @@ -583,7 +578,9 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ tscDebug("consumer:%" PRId64 ", begin commit for topic %s, vgId:%d", tmq->consumerId, pTopic->topicName, pVg->vgId); - if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { + tscDebug("consumer: %ld, vg:%d, current %ld, committed %ld", tmq->consumerId, pVg->vgId, + pVg->currentOffset.version, pVg->committedOffset.version); if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { continue; } @@ -699,7 +696,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { tmqAskEp(tmq, true); taosTmrReset(tmqAssignAskEpTask, 1000, tmq, tmqMgmt.timer, &tmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { - tmqCommitInner2(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); + tmqCommitInner(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) { } else { @@ -888,12 +885,6 @@ FAIL: return NULL; } -#if 0 -int32_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { - return tmqCommitInner2(tmq, offsets, 0, async, tmq->commitCb, tmq->commitCbUserParam); -} -#endif - int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); @@ -967,7 +958,11 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = param.rspErr; if (code != 0) goto FAIL; + int32_t retryCnt = 0; while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + if (retryCnt++ > 10) { + goto FAIL; + } tscDebug("consumer not ready, retry"); taosMsleep(500); } @@ -1006,8 +1001,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t epoch = pParam->epoch; taosMemoryFree(pParam); if (code != 0) { - tscWarn("msg discard from vgId:%d, epoch %d, code:%x", vgId, epoch, code); + tscWarn("msg discard from vgId:%d, epoch %d, since %s", vgId, epoch, terrstr()); if (pMsg->pData) taosMemoryFree(pMsg->pData); + if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); + goto CREATE_MSG_FAIL; + } if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { @@ -1083,7 +1082,7 @@ CREATE_MSG_FAIL: return -1; } -bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { +bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); @@ -1112,10 +1111,10 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); char buf[80]; - tFormatOffset(buf, 80, &pVgCur->currentOffsetNew); + tFormatOffset(buf, 80, &pVgCur->currentOffset); tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); - taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffsetNew, sizeof(STqOffsetVal)); + taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(STqOffsetVal)); } } } @@ -1142,7 +1141,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { SMqClientVg clientVg = { .pollCnt = 0, - .currentOffsetNew = offsetNew, + .currentOffset = offsetNew, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, @@ -1166,93 +1165,6 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { return set; } -#if 0 -bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { - /*printf("call update ep %d\n", epoch);*/ - bool set = false; - int32_t topicNumGet = taosArrayGetSize(pRsp->topics); - char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; - tscDebug("consumer:%" PRId64 ", update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, - topicNumGet); - SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); - if (newTopics == NULL) { - return false; - } - SHashObj* pHash = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); - if (pHash == NULL) { - taosArrayDestroy(newTopics); - return false; - } - - // find topic, build hash - for (int32_t i = 0; i < topicNumGet; i++) { - SMqClientTopic topic = {0}; - SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); - topic.schema = pTopicEp->schema; - taosHashClear(pHash); - topic.topicName = strdup(pTopicEp->topic); - tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN); - - tscDebug("consumer:%" PRId64 ", update topic: %s", tmq->consumerId, topic.topicName); - int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); - for (int32_t j = 0; j < topicNumCur; j++) { - // find old topic - SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); - if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) { - int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); - tscDebug("consumer:%" PRId64 ", new vg num: %d", tmq->consumerId, vgNumCur); - if (vgNumCur == 0) break; - for (int32_t k = 0; k < vgNumCur; k++) { - SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); - sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); - tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey); - taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); - } - break; - } - } - - int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); - topic.vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); - for (int32_t j = 0; j < vgNumGet; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); - int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); - int64_t offset = pVgEp->offset; - tscDebug("consumer:%" PRId64 ", (epoch %d) original offset of vgId:%d is %" PRId64, tmq->consumerId, epoch, pVgEp->vgId, offset); - if (pOffset != NULL) { - offset = *pOffset; - tscDebug("consumer:%" PRId64 ", (epoch %d) receive offset of vgId:%d, full key is %s", tmq->consumerId, epoch, pVgEp->vgId, - vgKey); - } - tscDebug("consumer:%" PRId64 ", (epoch %d) offset of vgId:%d updated to %" PRId64, tmq->consumerId, epoch, pVgEp->vgId, offset); - SMqClientVg clientVg = { - .pollCnt = 0, - .currentOffset = offset, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet, - .vgStatus = TMQ_VG_STATUS__IDLE, - .vgSkipCnt = 0, - }; - taosArrayPush(topic.vgs, &clientVg); - set = true; - } - taosArrayPush(newTopics, &topic); - } - if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); - taosHashCleanup(pHash); - tmq->clientTopics = newTopics; - - if (taosArrayGetSize(tmq->clientTopics) == 0) - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC); - else - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); - - atomic_store_32(&tmq->epoch, epoch); - return set; -} -#endif - int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; @@ -1278,7 +1190,7 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); /*printf("rsp epoch %" PRId64 " sz %" PRId64 "\n", rsp.epoch, rsp.topics->size);*/ /*printf("tmq epoch %" PRId64 " sz %" PRId64 "\n", tmq->epoch, tmq->clientTopics->size);*/ - tmqUpdateEp2(tmq, head->epoch, &rsp); + tmqUpdateEp(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM); @@ -1430,7 +1342,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; /*pReq->currentOffset = reqOffset;*/ - pReq->reqOffset = pVg->currentOffsetNew; + pReq->reqOffset = pVg->currentOffset; pReq->reqId = generateRequestId(); pReq->useSnapshot = tmq->useSnapshot; @@ -1534,7 +1446,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { /*printf("send poll\n");*/ char offsetFormatBuf[80]; - tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffsetNew); + tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffset); tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64, tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); /*printf("send vgId:%d %" PRId64 "\n", pVg->vgId, pVg->currentOffset);*/ @@ -1552,7 +1464,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) { SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; - tmqUpdateEp2(tmq, rspWrapper->epoch, rspMsg); + tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg); /*tmqClearUnhandleMsg(tmq);*/ *pReset = true; } else { @@ -1586,7 +1498,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, * rspMsg->msg.rspOffset);*/ - pVg->currentOffsetNew = pollRspWrapper->dataRsp.rspOffset; + pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); if (pollRspWrapper->dataRsp.blockNum == 0) { taosFreeQitem(pollRspWrapper); @@ -1609,8 +1521,8 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, * rspMsg->msg.rspOffset);*/ - pVg->currentOffsetNew.version = pollRspWrapper->metaRsp.rspOffset; - pVg->currentOffsetNew.type = TMQ_OFFSET__LOG; + pVg->currentOffset.version = pollRspWrapper->metaRsp.rspOffset; + pVg->currentOffset.type = TMQ_OFFSET__LOG; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); @@ -1653,6 +1565,17 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { return NULL; } + if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { + int32_t retryCnt = 0; + while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + if (retryCnt++ > 10) { + return NULL; + } + tscDebug("consumer not ready, retry"); + taosMsleep(500); + } + } + while (1) { tmqHandleAllDelayedTask(tmq); if (tmqPollImpl(tmq, timeout) < 0) return NULL; @@ -3381,10 +3304,10 @@ int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { // - tmqCommitInner2(tmq, msg, 0, 1, cb, param); + tmqCommitInner(tmq, msg, 0, 1, cb, param); } int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) { // - return tmqCommitInner2(tmq, msg, 0, 0, NULL, NULL); + return tmqCommitInner(tmq, msg, 0, 0, NULL, NULL); } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 675d3bbfdd..edd718090f 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -16,8 +16,8 @@ #include "systable.h" #include "taos.h" #include "tdef.h" -#include "types.h" #include "tgrant.h" +#include "types.h" #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) @@ -243,8 +243,8 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, -// {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, -// {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, + // {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, + // {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema)}, {TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema)}, {TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, @@ -284,7 +284,6 @@ static const SSysDbTableSchema consumerSchema[] = { {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "topics", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 39b57f2966..3cdc53cd2c 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -131,8 +131,9 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId, mndConsumerStatusName(pConsumer->status)); - if (pConsumer->status != MQ_CONSUMER_STATUS__READY) { + if (pConsumer->status != MQ_CONSUMER_STATUS__LOST_REBD) { mndReleaseConsumer(pMnode, pConsumer); + terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } @@ -275,6 +276,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int32_t status = atomic_load_32(&pConsumer->status); if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + mInfo("try to recover consumer %ld", consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -305,15 +307,14 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); -#if 1 atomic_store_32(&pConsumer->hbStatus, 0); -#endif // 1. check consumer status int32_t status = atomic_load_32(&pConsumer->status); #if 1 if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + mInfo("try to recover consumer %ld", consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -326,6 +327,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { #endif if (status != MQ_CONSUMER_STATUS__READY) { + mInfo("consumer %ld not ready, status: %s", consumerId, mndConsumerStatusName(status)); terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } @@ -939,10 +941,6 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * colDataAppend(pColInfo, numOfRows, NULL, true); } - // pid - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->pid, true); - // end point pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 4c6320ecb5..bce2b642e8 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -157,7 +157,7 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); int32_t tqProcessCheckAlterInfoReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); -int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen, int64_t ver); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 02ca248054..112543e340 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -183,7 +183,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con return 0; } -int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen, int64_t ver) { STqOffset offset = {0}; SDecoder decoder; tDecoderInit(&decoder, msg, msgLen); @@ -302,6 +302,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqError("tmq poll: consumer handle mismatch for consumer:%" PRId64 ", in vgId:%d, subkey %s, handle consumer id %" PRId64, consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0cb66484c0..2d181c5458 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -204,7 +204,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp break; case TDMT_VND_MQ_COMMIT_OFFSET: if (tqProcessOffsetCommitReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), - pMsg->contLen - sizeof(SMsgHead)) < 0) { + pMsg->contLen - sizeof(SMsgHead), version) < 0) { goto _err; } break; diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 7dc8d822e9..3776cb261f 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -44,7 +44,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); -SStreamQueueItem* streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); void streamFreeQitem(SStreamQueueItem* data); #ifdef __cplusplus diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 54014d7df9..c584c2a15f 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -125,7 +125,7 @@ void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) { } } -SStreamQueueItem* streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { ASSERT(elem); if (dst->type == STREAM_INPUT__DATA_BLOCK && elem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; @@ -172,8 +172,8 @@ void streamFreeQitem(SStreamQueueItem* data) { int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { - void* data = taosArrayGetP(pMerge->reqs, i); - taosMemoryFree(data); + void* dataStr = taosArrayGetP(pMerge->reqs, i); + taosMemoryFree(dataStr); taosMemoryFree(pRef); } } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9d98afa65a..c78ff0756f 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -370,80 +370,6 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat return 0; } -int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { - void* buf = NULL; - int32_t code = -1; - int32_t blockNum = taosArrayGetSize(data->blocks); - ASSERT(blockNum != 0); - - SStreamDispatchReq req = { - .streamId = pTask->streamId, - .dataSrcVgId = data->srcVgId, - .upstreamTaskId = pTask->taskId, - .upstreamChildId = pTask->selfChildId, - .upstreamNodeId = pTask->nodeId, - .blockNum = blockNum, - }; - - req.data = taosArrayInit(blockNum, sizeof(void*)); - req.dataLen = taosArrayInit(blockNum, sizeof(int32_t)); - if (req.data == NULL || req.dataLen == NULL) { - goto FAIL; - } - for (int32_t i = 0; i < blockNum; i++) { - SSDataBlock* pDataBlock = taosArrayGet(data->blocks, i); - if (streamAddBlockToDispatchMsg(pDataBlock, &req) < 0) { - goto FAIL; - } - } - int32_t vgId = 0; - int32_t downstreamTaskId = 0; - // find ep - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - vgId = pTask->fixedEpDispatcher.nodeId; - *ppEpSet = &pTask->fixedEpDispatcher.epSet; - downstreamTaskId = pTask->fixedEpDispatcher.taskId; - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - // TODO get ctbName for each block - SSDataBlock* pBlock = taosArrayGet(data->blocks, 0); - char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId); - // TODO: get hash function by hashMethod - - // get groupId, compute hash value - uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); - - // get node - // TODO: optimize search process - SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t sz = taosArrayGetSize(vgInfo); - for (int32_t i = 0; i < sz; i++) { - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - ASSERT(pVgInfo->vgId > 0); - if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { - vgId = pVgInfo->vgId; - downstreamTaskId = pVgInfo->taskId; - *ppEpSet = &pVgInfo->epSet; - break; - } - } - } - - ASSERT(vgId > 0 || vgId == SNODE_HANDLE); - req.taskId = downstreamTaskId; - - qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, - downstreamTaskId, vgId); - - streamDispatchOneReq(pTask, &req, vgId, *ppEpSet); - - code = 0; -FAIL: - if (code < 0 && buf) rpcFreeCont(buf); - if (req.data) taosArrayDestroyP(req.data, (FDelete)taosMemoryFree); - if (req.dataLen) taosArrayDestroy(req.dataLen); - return code; -} - int32_t streamDispatch(SStreamTask* pTask) { ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); @@ -461,7 +387,7 @@ int32_t streamDispatch(SStreamTask* pTask) { } ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - qDebug("stream continue dispatching: task %d", pTask->taskId); + qDebug("stream dispatching: task %d", pTask->taskId); int32_t code = 0; if (streamDispatchAllBlocks(pTask, pBlock) < 0) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 72249c5181..06ca26f029 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -82,17 +82,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* return 0; } -#if 0 -static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock* pBlock) { - ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - int32_t childId = pBlock->childId; - int64_t ver = pBlock->sourceVer; - SStreamChildEpInfo* pChildInfo = taosArrayGetP(pTask->childEpInfo, childId); - /*pChildInfo-> = ver;*/ - return 0; -} -#endif - int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch) { ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); @@ -150,10 +139,11 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch) return 0; } + // TODO: handle version int32_t streamExecForAll(SStreamTask* pTask) { while (1) { - int32_t cnt = 1; + int32_t batchCnt = 1; void* data = NULL; while (1) { SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); @@ -169,13 +159,12 @@ int32_t streamExecForAll(SStreamTask* pTask) { } } else { void* newRet; - if ((newRet = streamAppendQueueItem(data, qItem)) == NULL) { + if ((newRet = streamMergeQueueItem(data, qItem)) == NULL) { streamQueueProcessFail(pTask->inputQueue); break; } else { - cnt++; + batchCnt++; data = newRet; - /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ streamQueueProcessSuccess(pTask->inputQueue); } } @@ -198,16 +187,14 @@ int32_t streamExecForAll(SStreamTask* pTask) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); + qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, batchCnt); streamTaskExecImpl(pTask, data, pRes); qDebug("stream task %d exec end", pTask->taskId); if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); if (qRes == NULL) { - // TODO log failed ver - streamQueueProcessFail(pTask->inputQueue); - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); streamFreeQitem(data); return -1; } @@ -218,17 +205,18 @@ int32_t streamExecForAll(SStreamTask* pTask) { SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; + } else if (((SStreamQueueItem*)data)->type == STREAM_INPUT__MERGED_SUBMIT) { + SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)data; + qRes->childId = pTask->selfChildId; + qRes->sourceVer = pMerged->ver; } if (streamTaskOutput(pTask, qRes) < 0) { - // TODO save failed ver - /*streamQueueProcessFail(pTask->inputQueue);*/ taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - taosFreeQitem(qRes); streamFreeQitem(data); + taosFreeQitem(qRes); return -1; } - /*streamQueueProcessSuccess(pTask->inputQueue);*/ } else { taosArrayDestroy(pRes); } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index ad571a9e82..fad3977a21 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -421,6 +421,8 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); + wDebug("vgId:%d, wal write log %ld, msgType: %s", pWal->cfg.vgId, index, TMSG_INFO(msgType)); + if (taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 63c414386b..ab4fb551fe 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -76,8 +76,8 @@ int32_t tsem_wait(tsem_t* sem) { int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { struct timespec ts, rel; - FILETIME ft_before, ft_after; - int rc; + FILETIME ft_before, ft_after; + int rc; rel.tv_sec = 0; rel.tv_nsec = nanosecs; @@ -217,7 +217,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int e = errno; // if (e == EEXIST) continue; // if (e == EINTR) continue; -// fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, +// fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem, // e, strerror(e)); // abort(); // } while (p->sem == SEM_FAILED); @@ -231,7 +232,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value); // if (ret != KERN_SUCCESS) { -// fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // // we fail-fast here, because we have less-doc about semaphore_create for the moment // abort(); @@ -258,8 +260,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // struct tsem_s *p = *sem; // if (!p->valid) { -// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); -// abort(); +// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); abort(); // } // #ifdef SEM_USE_PTHREAD // if (taosThreadMutexLock(&p->lock)) { @@ -270,7 +272,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // p->val -= 1; // if (p->val < 0) { // if (taosThreadCondWait(&p->cond, &p->lock)) { -// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // abort(); // } @@ -297,8 +300,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // struct tsem_s *p = *sem; // if (!p->valid) { -// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); -// abort(); +// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); abort(); // } // #ifdef SEM_USE_PTHREAD // if (taosThreadMutexLock(&p->lock)) { @@ -309,7 +312,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // p->val += 1; // if (p->val <= 0) { // if (taosThreadCondSignal(&p->cond)) { -// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // abort(); // } @@ -332,7 +336,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int tsem_destroy(tsem_t *sem) { // // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); // if (!*sem) { -// // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); +// // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); // // abort(); // return 0; // } @@ -370,7 +375,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int r = sem_unlink(name); // if (r) { // int e = errno; -// fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, +// fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem, // e, strerror(e)); // abort(); // } @@ -385,225 +391,189 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // *sem = NULL; // return 0; // } -typedef struct -{ - pthread_mutex_t count_lock; - pthread_cond_t count_bump; - unsigned int count; -}bosal_sem_t; +typedef struct { + pthread_mutex_t count_lock; + pthread_cond_t count_bump; + unsigned int count; +} bosal_sem_t; -int tsem_init(tsem_t *psem, int flags, unsigned int count) -{ - bosal_sem_t *pnewsem; - int result; +int tsem_init(tsem_t *psem, int flags, unsigned int count) { + bosal_sem_t *pnewsem; + int result; - pnewsem = (bosal_sem_t *)malloc(sizeof(bosal_sem_t)); - if (! pnewsem) - { - return -1; - } - result = pthread_mutex_init(&pnewsem->count_lock, NULL); - if (result) - { - free(pnewsem); - return result; - } - result = pthread_cond_init(&pnewsem->count_bump, NULL); - if (result) - { - pthread_mutex_destroy(&pnewsem->count_lock); - free(pnewsem); - return result; - } - pnewsem->count = count; - *psem = (tsem_t)pnewsem; - return 0; + pnewsem = (bosal_sem_t *)malloc(sizeof(bosal_sem_t)); + if (!pnewsem) { + return -1; + } + result = pthread_mutex_init(&pnewsem->count_lock, NULL); + if (result) { + free(pnewsem); + return result; + } + result = pthread_cond_init(&pnewsem->count_bump, NULL); + if (result) { + pthread_mutex_destroy(&pnewsem->count_lock); + free(pnewsem); + return result; + } + pnewsem->count = count; + *psem = (tsem_t)pnewsem; + return 0; } -int tsem_destroy(tsem_t *psem) -{ - bosal_sem_t *poldsem; +int tsem_destroy(tsem_t *psem) { + bosal_sem_t *poldsem; - if (! psem) - { - return EINVAL; - } - poldsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + poldsem = (bosal_sem_t *)*psem; - pthread_mutex_destroy(&poldsem->count_lock); - pthread_cond_destroy(&poldsem->count_bump); - free(poldsem); - return 0; + pthread_mutex_destroy(&poldsem->count_lock); + pthread_cond_destroy(&poldsem->count_bump); + free(poldsem); + return 0; } -int tsem_post(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_post(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - pxsem->count = pxsem->count + 1; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + pxsem->count = pxsem->count + 1; - xresult = pthread_cond_signal(&pxsem->count_bump); + xresult = pthread_cond_signal(&pxsem->count_bump); - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_trywait(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_trywait(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count > 0) - { - pxsem->count--; - } - else - { - xresult = EAGAIN; - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + if (pxsem->count > 0) { + pxsem->count--; + } else { + xresult = EAGAIN; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_wait(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_wait(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count == 0) - { - xresult = pthread_cond_wait(&pxsem->count_bump, &pxsem->count_lock); + if (pxsem->count == 0) { + xresult = pthread_cond_wait(&pxsem->count_bump, &pxsem->count_lock); + } + if (!xresult) { + if (pxsem->count > 0) { + pxsem->count--; } - if (! xresult) - { - if (pxsem->count > 0) - { - pxsem->count--; - } - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_timewait(tsem_t *psem, int64_t nanosecs) -{ +int tsem_timewait(tsem_t *psem, int64_t nanosecs) { struct timespec abstim = { .tv_sec = 0, .tv_nsec = nanosecs, }; - bosal_sem_t *pxsem; - int result, xresult; + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count == 0) - { - xresult = pthread_cond_timedwait(&pxsem->count_bump, &pxsem->count_lock, &abstim); + if (pxsem->count == 0) { + xresult = pthread_cond_timedwait(&pxsem->count_bump, &pxsem->count_lock, &abstim); + } + if (!xresult) { + if (pxsem->count > 0) { + pxsem->count--; } - if (! xresult) - { - if (pxsem->count > 0) - { - pxsem->count--; - } - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -bool taosCheckPthreadValid(TdThread thread) { +bool taosCheckPthreadValid(TdThread thread) { int32_t ret = taosThreadKill(thread, 0); if (ret == ESRCH) return false; if (ret == EINVAL) return false; // alive return true; - } +} int64_t taosGetSelfPthreadId() { TdThread thread = taosThreadSelf(); @@ -650,7 +620,13 @@ int64_t taosGetSelfPthreadId() { int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; } void taosResetPthread(TdThread* thread) { *thread = 0; } bool taosComparePthread(TdThread first, TdThread second) { return first == second; } -int32_t taosGetPId() { return getpid(); } + +int32_t taosGetPId() { + static __thread int32_t pid = 0; + if (pid != 0) return pid; + pid = getpid(); + return pid; +} int32_t taosGetAppName(char* name, int32_t* len) { const char* self = "/proc/self/exe"; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4780d85a30..93b91739a3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -621,6 +621,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Invalid index file" //tmq TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") #ifdef TAOS_ERROR_C }; From b4c8685766c1a8ca6d50d6fa63da2cb3e13ba5e4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 10 Aug 2022 17:17:30 +0800 Subject: [PATCH 65/65] refactor(sync): add leader, follower call back --- include/libs/sync/sync.h | 3 +++ source/dnode/mnode/impl/src/mndSync.c | 14 ++++++++++++++ source/dnode/vnode/src/vnd/vnodeSync.c | 14 ++++++++++++++ source/libs/sync/src/syncMain.c | 12 +++++++++++- 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 35e79cf45d..aa563343f8 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -129,6 +129,9 @@ typedef struct SSyncFSM { void (*FpReConfigCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta); void (*FpLeaderTransferCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); + void (*FpBecomeLeaderCb)(struct SSyncFSM* pFsm); + void (*FpBecomeFollowerCb)(struct SSyncFSM* pFsm); + int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot, void* pReaderParam, void** ppReader); int32_t (*FpGetSnapshotInfo)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index b8ff7be46a..37d5aeb62d 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -166,6 +166,18 @@ void mndLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cb mDebug("vgId:1, mnode leader transfer finish"); } +static void mndBecomeFollower(struct SSyncFSM *pFsm) { + SMnode *pMnode = pFsm->data; + mDebug("vgId:1, become follower"); + + // clear old leader resource +} + +static void mndBecomeLeader(struct SSyncFSM *pFsm) { + SMnode *pMnode = pFsm->data; + mDebug("vgId:1, become leader"); +} + SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pMnode; @@ -175,6 +187,8 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { pFsm->FpRestoreFinishCb = mndRestoreFinish; pFsm->FpLeaderTransferCb = mndLeaderTransfer; pFsm->FpReConfigCb = mndReConfig; + pFsm->FpBecomeLeaderCb = mndBecomeLeader; + pFsm->FpBecomeFollowerCb = mndBecomeFollower; pFsm->FpGetSnapshot = mndSyncGetSnapshot; pFsm->FpGetSnapshotInfo = mndSyncGetSnapshotInfo; pFsm->FpSnapshotStartRead = mndSnapshotStartRead; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index c525d09057..7ac124fdd3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -672,6 +672,18 @@ static void vnodeRestoreFinish(struct SSyncFSM *pFsm) { vDebug("vgId:%d, sync restore finished", pVnode->config.vgId); } +static void vnodeBecomeFollower(struct SSyncFSM *pFsm) { + SVnode *pVnode = pFsm->data; + vDebug("vgId:%d, become follower", pVnode->config.vgId); + + // clear old leader resource +} + +static void vnodeBecomeLeader(struct SSyncFSM *pFsm) { + SVnode *pVnode = pFsm->data; + vDebug("vgId:%d, become leader", pVnode->config.vgId); +} + static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pVnode; @@ -681,6 +693,8 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshot; pFsm->FpRestoreFinishCb = vnodeRestoreFinish; pFsm->FpLeaderTransferCb = vnodeLeaderTransfer; + pFsm->FpBecomeLeaderCb = vnodeBecomeLeader; + pFsm->FpBecomeFollowerCb = vnodeBecomeFollower; pFsm->FpReConfigCb = vnodeSyncReconfig; pFsm->FpSnapshotStartRead = vnodeSnapshotStartRead; pFsm->FpSnapshotStopRead = vnodeSnapshotStopRead; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 20b4b23bf7..5f3ff3015c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2028,6 +2028,11 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // reset elect timer syncNodeResetElectTimer(pSyncNode); + // call back + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpBecomeFollowerCb != NULL) { + pSyncNode->pFsm->FpBecomeFollowerCb(pSyncNode->pFsm); + } + // trace log do { int32_t debugStrLen = strlen(debugStr); @@ -2109,6 +2114,11 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // start heartbeat timer syncNodeStartHeartbeatTimer(pSyncNode); + // call back + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpBecomeLeaderCb != NULL) { + pSyncNode->pFsm->FpBecomeLeaderCb(pSyncNode->pFsm); + } + // trace log do { int32_t debugStrLen = strlen(debugStr); @@ -3100,7 +3110,7 @@ void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntries syncNodeEventLog(pSyncNode, logBuf); } - void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { +void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);