From a62a7cc41f709c829e7ebe8ef0bbe0f3807f1307 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 1 Aug 2022 21:05:44 +0800 Subject: [PATCH 01/96] enh: aggregate and window add field for merge data blocks --- include/libs/nodes/plannodes.h | 2 ++ source/libs/nodes/src/nodesCodeFuncs.c | 17 ++++++++-- source/libs/planner/src/planLogicCreater.c | 20 +++++++----- source/libs/planner/src/planPhysiCreater.c | 3 ++ source/libs/planner/src/planSpliter.c | 4 +++ source/libs/planner/test/planGroupByTest.cpp | 34 ++++++++++---------- 6 files changed, 53 insertions(+), 27 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 2f6bb603c1..6f25cc45fb 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -377,6 +377,7 @@ typedef struct SAggPhysiNode { SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function SNodeList* pGroupKeys; SNodeList* pAggFuncs; + bool mergeDataBlock; } SAggPhysiNode; typedef struct SDownstreamSourceNode { @@ -415,6 +416,7 @@ typedef struct SWinodwPhysiNode { int8_t igExpired; EOrder inputTsOrder; EOrder outputTsOrder; + bool mergeDataBlock; } SWinodwPhysiNode; typedef struct SIntervalPhysiNode { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 9d15b01acf..74659179a9 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1773,6 +1773,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { static const char* jkAggPhysiPlanExprs = "Exprs"; static const char* jkAggPhysiPlanGroupKeys = "GroupKeys"; static const char* jkAggPhysiPlanAggFuncs = "AggFuncs"; +static const char* jkAggPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) { const SAggPhysiNode* pNode = (const SAggPhysiNode*)pObj; @@ -1787,6 +1788,9 @@ static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkAggPhysiPlanAggFuncs, pNode->pAggFuncs); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanMergeDataBlock, pNode->mergeDataBlock); + } return code; } @@ -1804,6 +1808,9 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkAggPhysiPlanAggFuncs, &pNode->pAggFuncs); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkAggPhysiPlanMergeDataBlock, &pNode->mergeDataBlock); + } return code; } @@ -1936,8 +1943,9 @@ static const char* jkWindowPhysiPlanTsEnd = "TsEnd"; static const char* jkWindowPhysiPlanTriggerType = "TriggerType"; static const char* jkWindowPhysiPlanWatermark = "Watermark"; static const char* jkWindowPhysiPlanIgnoreExpired = "IgnoreExpired"; -static const char* jkWindowPhysiPlanInputTsOrder = "inputTsOrder"; +static const char* jkWindowPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkWindowPhysiPlanOutputTsOrder = "outputTsOrder"; +static const char* jkWindowPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; @@ -1970,6 +1978,9 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanOutputTsOrder, pNode->outputTsOrder); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkWindowPhysiPlanMergeDataBlock, pNode->mergeDataBlock); + } return code; } @@ -2005,6 +2016,9 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkWindowPhysiPlanOutputTsOrder, pNode->outputTsOrder, code); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkWindowPhysiPlanMergeDataBlock, &pNode->mergeDataBlock); + } return code; } @@ -2147,7 +2161,6 @@ static int32_t jsonToPhysiSessionWindowNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysiWindowNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkSessionWindowPhysiPlanGap, pNode->gap, code); - ; } return code; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b51624336b..a799b2feb6 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -468,6 +468,10 @@ static int32_t createGroupKeysFromPartKeys(SNodeList* pPartKeys, SNodeList** pOu return TSDB_CODE_SUCCESS; } +static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { + return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; +} + static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; @@ -480,7 +484,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->hasLastRow = pSelect->hasLastRowFunc; pAgg->hasTimeLineFunc = pSelect->hasTimeLineFunc; - pAgg->node.groupAction = GROUP_ACTION_SET; + pAgg->node.groupAction = getGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = pAgg->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; @@ -551,7 +555,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc; pIdfRowsFunc->isUniqueFunc = pSelect->hasUniqueFunc; pIdfRowsFunc->isTimeLineFunc = pSelect->hasTimeLineFunc; - pIdfRowsFunc->node.groupAction = GROUP_ACTION_KEEP; + pIdfRowsFunc->node.groupAction = getGroupAction(pCxt, pSelect); pIdfRowsFunc->node.requireDataOrder = pIdfRowsFunc->isTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; pIdfRowsFunc->node.resultDataOrder = pIdfRowsFunc->node.requireDataOrder; @@ -586,7 +590,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p return TSDB_CODE_OUT_OF_MEMORY; } - pInterpFunc->node.groupAction = GROUP_ACTION_KEEP; + pInterpFunc->node.groupAction = getGroupAction(pCxt, pSelect); pInterpFunc->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder; @@ -662,7 +666,7 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo } pWindow->winType = WINDOW_TYPE_STATE; - pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pWindow->pStateExpr = nodesCloneNode(pState->pExpr); @@ -685,7 +689,7 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW pWindow->winType = WINDOW_TYPE_SESSION; pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i; pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? SESSION_ALGO_STREAM_SINGLE : SESSION_ALGO_MERGE; - pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; @@ -714,7 +718,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; - pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); pWindow->node.requireDataOrder = pSelect->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_IN_BLOCK; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; @@ -762,7 +766,7 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return TSDB_CODE_OUT_OF_MEMORY; } - pFill->node.groupAction = GROUP_ACTION_KEEP; + pFill->node.groupAction = getGroupAction(pCxt, pSelect); pFill->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pFill->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pFill->inputTsOrder = ORDER_ASC; @@ -936,7 +940,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = GROUP_ACTION_SET; + pAgg->node.groupAction = GROUP_ACTION_CLEAR; pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 3771586b34..af3d4a9577 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -825,6 +825,8 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, return TSDB_CODE_OUT_OF_MEMORY; } + pAgg->mergeDataBlock = (GROUP_ACTION_KEEP == pAggLogicNode->node.groupAction ? false : true); + SNodeList* pPrecalcExprs = NULL; SNodeList* pGroupKeys = NULL; SNodeList* pAggFuncs = NULL; @@ -1091,6 +1093,7 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pWindow->igExpired = pWindowLogicNode->igExpired; pWindow->inputTsOrder = pWindowLogicNode->inputTsOrder; pWindow->outputTsOrder = pWindowLogicNode->outputTsOrder; + pWindow->mergeDataBlock = (GROUP_ACTION_KEEP == pWindowLogicNode->node.groupAction ? false : true); SNodeList* pPrecalcExprs = NULL; SNodeList* pFuncs = NULL; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 74c78b970d..6ddb6d90c0 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -393,6 +393,8 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic code = TSDB_CODE_OUT_OF_MEMORY; } + pPartWin->node.groupAction = GROUP_ACTION_KEEP; + if (TSDB_CODE_SUCCESS == code) { pMergeWindow->node.pTargets = pTargets; pMergeWindow->node.pConditions = pConditions; @@ -723,6 +725,8 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO code = TSDB_CODE_OUT_OF_MEMORY; } + pPartAgg->node.groupAction = GROUP_ACTION_KEEP; + if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { pPartAgg->pGroupKeys = pGroupKeys; code = createColumnByRewriteExprs(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets); diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index 657d8a6244..a553d3addc 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -32,6 +32,22 @@ TEST_F(PlanGroupByTest, basic) { run("SELECT c1 + c3, SUM(c4 * c5) FROM t1 WHERE CONCAT(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); run("SELECT SUM(CEIL(c1)) FROM t1 GROUP BY CEIL(c1)"); + + run("SELECT COUNT(*) FROM st1"); + + run("SELECT c1 FROM st1 GROUP BY c1"); + + run("SELECT COUNT(*) FROM st1 GROUP BY c1"); + + run("SELECT SUM(c1) FROM st1 GROUP BY c2 HAVING SUM(c1) IS NOT NULL"); +} + +TEST_F(PlanGroupByTest, withPartitionBy) { + useDb("root", "test"); + + run("SELECT LAST(ts), TBNAME FROM st1 PARTITION BY TBNAME"); + + run("SELECT COUNT(*) FROM st1 PARTITION BY c2 GROUP BY c1"); } TEST_F(PlanGroupByTest, withOrderBy) { @@ -43,14 +59,12 @@ TEST_F(PlanGroupByTest, withOrderBy) { // run("SELECT COUNT(*), SUM(c1) a FROM t1 ORDER BY a"); } -TEST_F(PlanGroupByTest, aggFunc) { +TEST_F(PlanGroupByTest, multiResFunc) { useDb("root", "test"); run("SELECT LAST(*), FIRST(*) FROM t1"); run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1"); - - run("SELECT SUM(10), COUNT(c1) FROM t1 GROUP BY c2"); } TEST_F(PlanGroupByTest, selectFunc) { @@ -67,17 +81,3 @@ TEST_F(PlanGroupByTest, selectFunc) { run("SELECT MAX(c1), c2 FROM t1 GROUP BY c3"); run("SELECT MAX(c1), t1.* FROM t1 GROUP BY c3"); } - -TEST_F(PlanGroupByTest, stable) { - useDb("root", "test"); - - run("SELECT COUNT(*) FROM st1"); - - run("SELECT c1 FROM st1 GROUP BY c1"); - - run("SELECT COUNT(*) FROM st1 GROUP BY c1"); - - run("SELECT COUNT(*) FROM st1 PARTITION BY c2 GROUP BY c1"); - - run("SELECT SUM(c1) FROM st1 GROUP BY c2 HAVING SUM(c1) IS NOT NULL"); -} From 781c494c2be7d2209eaa18ae99bbafeccf18af32 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 13:58:54 +0800 Subject: [PATCH 02/96] refactor: merge data belongs to different data group. --- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/executorimpl.c | 76 +++++++++---------- source/libs/executor/src/timewindowoperator.c | 1 + 3 files changed, 37 insertions(+), 44 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 577f9772be..e89459e555 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -519,6 +519,7 @@ typedef struct SBlockDistInfo { typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; SSDataBlock* pRes; + bool mergeResultBlock; } SOptrBasicInfo; typedef struct SIntervalAggOperatorInfo { @@ -593,7 +594,6 @@ typedef struct SAggOperatorInfo { uint64_t groupId; SGroupResInfo groupResInfo; SExprSupp scalarExprSup; - SNode *pCondition; } SAggOperatorInfo; @@ -891,7 +891,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, - int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); + int32_t numOfScalarExpr, bool mergeResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e52cbf40a9..a7b843d9c2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -136,9 +136,8 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, void operatorDummyCloseFn(void* param, int32_t numOfCols) {} -static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, - SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset, - SqlFunctionCtx* pCtx, int32_t numOfExprs); +static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); @@ -1501,19 +1500,24 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi return 0; } -int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, - SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, - int32_t numOfExprs) { - int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); - int32_t start = pGroupResInfo->index; +int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo) { + SExprInfo* pExprInfo = pSup->pExprInfo; + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; - for (int32_t i = start; i < numOfRows; i += 1) { + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + + for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); - doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset); + doUpdateNumOfRows(pRow, numOfExprs, rowEntryOffset); + + // no results, continue to check the next one if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; releaseBufPage(pBuf, page); @@ -1531,6 +1535,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); releaseBufPage(pBuf, page); break; } @@ -1540,7 +1545,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI for (int32_t j = 0; j < numOfExprs; ++j) { int32_t slotId = pExprInfo[j].base.resSchema.slotId; - pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowCellOffset); + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (pCtx[j].fpSet.finalize) { #ifdef BUF_PAGE_DEBUG qDebug("\npage_finalize %d", numOfExprs); @@ -1573,26 +1578,19 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI releaseBufPage(pBuf, page); pBlock->info.rows += pRow->numOfRows; - // if (pBlock->info.rows >= pBlock->info.capacity) { // output buffer is full - // break; - // } } qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.groupId); + blockDataUpdateTsWindow(pBlock, 0); return 0; } void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { - SExprInfo* pExprInfo = pOperator->exprSupp.pExprInfo; - int32_t numOfExprs = pOperator->exprSupp.numOfExprs; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int32_t* rowCellOffset = pOperator->exprSupp.rowEntryInfoOffset; - SSDataBlock* pBlock = pbInfo->pRes; - SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; + SSDataBlock* pBlock = pbInfo->pRes; // set output datablock version pBlock->info.version = pTaskInfo->version; @@ -1604,30 +1602,22 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG // clear the existed group id pBlock->info.groupId = 0; - doCopyToSDataBlock(pTaskInfo, pBlock, pExprInfo, pBuf, pGroupResInfo, rowCellOffset, pCtx, numOfExprs); -} - -static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, - int32_t* rowEntryInfoOffset) { - // update the number of result for each, only update the number of rows for the corresponding window result. - // if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - // return; - // } -#if 0 - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow* pResult = pResultRowInfo->pResult[i]; - - for (int32_t j = 0; j < numOfOutput; ++j) { - int32_t functionId = pCtx[j].functionId; - if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ) { - continue; + if (!pbInfo->mergeResultBlock) { + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo); + } else { + while(hasRemainResults(pGroupResInfo)) { + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo); + if (pBlock->info.rows >= pOperator->resultInfo.threshold) { + break; } - SResultRowEntryInfo* pCell = getResultEntryInfo(pResult, j, rowEntryInfoOffset); - pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); + // clearing group id to continue to merge data that belong to different groups + pBlock->info.groupId = 0; } + + // clear the group id info in SSDataBlock, since the client does not need it + pBlock->info.groupId = 0; } -#endif } static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, int8_t compressed) { @@ -2979,6 +2969,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { break; } } + size_t rows = blockDataGetNumOfRows(pInfo->pRes); pOperator->resultInfo.totalRows += rows; @@ -3483,7 +3474,7 @@ void cleanupExprSupp(SExprSupp* pSupp) { SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, - int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo) { + int32_t numOfScalarExpr, bool mergeResult, SExecTaskInfo* pTaskInfo) { SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3505,6 +3496,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } + pInfo->binfo.mergeResultBlock = mergeResult; pInfo->groupId = UINT64_MAX; pInfo->pCondition = pCondition; pOperator->name = "TableAggregate"; @@ -4098,7 +4090,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pScalarExprInfo, numOfScalarExpr, pTaskInfo); } else { pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, - pScalarExprInfo, numOfScalarExpr, pTaskInfo); + pScalarExprInfo, numOfScalarExpr, pAggNode->mergeDataBlock, pTaskInfo); } } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 802e1f2306..c447d6b43b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1798,6 +1798,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->twAggSup = *pTwAggSupp; pInfo->ignoreExpiredData = pPhyNode->window.igExpired; pInfo->pCondition = pPhyNode->window.node.pConditions; + pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock; if (pPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; From 770cc74e7f220b2465b3c0402f2c2bd9844c51c6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 14:19:15 +0800 Subject: [PATCH 03/96] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8821dbd5a1..43210ced2a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -464,6 +464,11 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pBlock->info = binfo; ASSERT(binfo.uid != 0); + uint64_t* groupId = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); + if (groupId) { + pBlock->info.groupId = *groupId; + } + uint32_t status = 0; int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); @@ -476,11 +481,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - uint64_t* groupId = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); - if (groupId) { - pBlock->info.groupId = *groupId; - } - pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows; pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; From be51233c241d906c9525d38e95c1943fb7a77413 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 14:26:54 +0800 Subject: [PATCH 04/96] refactor: add dynamic prune for last partial. --- source/libs/function/src/builtins.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6383179fee..23e26f7716 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2343,6 +2343,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_LAST_PARTIAL, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLastPartial, + .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastFunction, From cbed4713afccd050c7d4189dd460b9517eb993d9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 14:34:50 +0800 Subject: [PATCH 05/96] refactor: do some internal refactor. --- source/libs/function/src/functionMgt.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 339ff3808b..72ba81c3a7 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -116,6 +116,11 @@ EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, STimeWindow* return TSDB_CODE_FAILED; } + const char* name = funcMgtBuiltins[funcId].name; + if ((strcmp(name, "_group_key") == 0) || (strcmp(name, "_select_value") == 0)) { + return FUNC_DATA_REQUIRED_NOT_LOAD; + } + if (funcMgtBuiltins[funcId].dynDataRequiredFunc == NULL) { return FUNC_DATA_REQUIRED_DATA_LOAD; } else { From 258d50306fa5295fa21bb9434bdcdfcd7f8d0e71 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 15:43:45 +0800 Subject: [PATCH 06/96] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/cachescanoperator.c | 2 +- source/libs/executor/src/executorimpl.c | 5 +- source/libs/executor/src/timewindowoperator.c | 124 +++++++++--------- 4 files changed, 63 insertions(+), 72 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index e89459e555..d24156bc60 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -554,8 +554,6 @@ typedef struct SMergeAlignedIntervalAggOperatorInfo { uint64_t groupId; // current groupId int64_t curTs; // current ts SSDataBlock* prefetchedBlock; - bool inputBlocksFinished; - SNode* pCondition; } SMergeAlignedIntervalAggOperatorInfo; @@ -906,7 +904,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - SExecTaskInfo* pTaskInfo); + bool mergeResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 56a5e253d8..94e4384b30 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -50,7 +50,7 @@ SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pScanNode, SRead STableListInfo* pTableList = &pTaskInfo->tableqinfoList; - initResultSizeInfo(&pOperator->resultInfo, 1024); + initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); pInfo->pUidList = taosArrayInit(4, sizeof(int64_t)); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a7b843d9c2..794e751dd2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3481,10 +3481,9 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - int32_t numOfRows = 1024; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, numOfRows); + initResultSizeInfo(&pOperator->resultInfo, 4096); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -4147,7 +4146,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; - pOptr = createMergeIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pTaskInfo); + pOptr = createMergeIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pIntervalPhyNode->window.mergeDataBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL == type) { int32_t children = 0; pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c447d6b43b..812428920a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4622,20 +4622,25 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR int32_t numOfOutput = pSup->numOfExprs; int64_t* tsCols = extractTsCol(pBlock, iaInfo); uint64_t tableGroupId = pBlock->info.groupId; - TSKEY currTs = getStartTsKey(&pBlock->info.window, tsCols); SResultRow* pResult = NULL; + TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); + // there is an result exists if (miaInfo->curTs != INT64_MIN) { ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); - if (currTs != miaInfo->curTs) { + + if (ts != miaInfo->curTs) { outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs); - miaInfo->curTs = INT64_MIN; + miaInfo->curTs = ts; } + } else { + miaInfo->curTs = ts; + ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); } STimeWindow win = {0}; - win.skey = currTs; + win.skey = miaInfo->curTs; win.ekey = taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1; @@ -4646,12 +4651,11 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - miaInfo->curTs = win.skey; int32_t currPos = startPos; STimeWindow currWin = win; while (++currPos < pBlock->info.rows) { - if (tsCols[currPos] == currTs) { + if (tsCols[currPos] == miaInfo->curTs) { continue; } @@ -4659,11 +4663,10 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder); - outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs); - miaInfo->curTs = INT64_MIN; + outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs); + miaInfo->curTs = tsCols[currPos]; - currTs = tsCols[currPos]; - currWin.skey = currTs; + currWin.skey = miaInfo->curTs; currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1; @@ -4680,14 +4683,6 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder); - - if (currPos >= pBlock->info.rows) { - // we need to see next block if exists - } else { - ASSERT(0); - outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs); - miaInfo->curTs = INT64_MIN; - } } static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { @@ -4703,59 +4698,56 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SSDataBlock* pRes = iaInfo->binfo.pRes; blockDataCleanup(pRes); - blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); - if (!miaInfo->inputBlocksFinished) { - SOperatorInfo* downstream = pOperator->pDownstream[0]; - int32_t scanFlag = MAIN_SCAN; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + int32_t scanFlag = MAIN_SCAN; - while (1) { - SSDataBlock* pBlock = NULL; - if (miaInfo->prefetchedBlock == NULL) { - pBlock = downstream->fpSet.getNextFn(downstream); - } else { - pBlock = miaInfo->prefetchedBlock; - miaInfo->groupId = pBlock->info.groupId; - miaInfo->prefetchedBlock = NULL; - } + while (1) { + SSDataBlock* pBlock = NULL; + if (miaInfo->prefetchedBlock == NULL) { + pBlock = downstream->fpSet.getNextFn(downstream); + } else { + pBlock = miaInfo->prefetchedBlock; + miaInfo->prefetchedBlock = NULL; - if (pBlock == NULL) { - // close last unfinalized time window - if (miaInfo->curTs != INT64_MIN) { - ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); - outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); - miaInfo->curTs = INT64_MIN; - } - - doSetOperatorCompleted(pOperator); - miaInfo->inputBlocksFinished = true; - break; - } - - if (!miaInfo->hasGroupId) { - miaInfo->hasGroupId = true; - miaInfo->groupId = pBlock->info.groupId; - } else if (miaInfo->groupId != pBlock->info.groupId) { - // if there are unclosed time window, close it firstly. - ASSERT(miaInfo->curTs != INT64_MIN); - outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); - miaInfo->prefetchedBlock = pBlock; - miaInfo->curTs = INT64_MIN; - break; - } - - getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag); - setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true); - doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); - doFilter(miaInfo->pCondition, pRes, NULL); - if (pRes->info.rows >= pOperator->resultInfo.capacity) { - break; - } + miaInfo->groupId = pBlock->info.groupId; } - pRes->info.groupId = miaInfo->groupId; + if (pBlock == NULL) { + // close last unfinalized time window + if (miaInfo->curTs != INT64_MIN) { + ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); + outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); + miaInfo->curTs = INT64_MIN; + } + + doSetOperatorCompleted(pOperator); + break; + } + + if (!miaInfo->hasGroupId) { + miaInfo->hasGroupId = true; + miaInfo->groupId = pBlock->info.groupId; + } else if (miaInfo->groupId != pBlock->info.groupId) { + // if there are unclosed time window, close it firstly. + ASSERT(miaInfo->curTs != INT64_MIN); + outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); + miaInfo->prefetchedBlock = pBlock; + miaInfo->curTs = INT64_MIN; + break; + } + + getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag); + setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true); + doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); + + doFilter(miaInfo->pCondition, pRes, NULL); + if (pRes->info.rows >= pOperator->resultInfo.capacity) { + break; + } } + pRes->info.groupId = miaInfo->groupId; miaInfo->hasGroupId = false; size_t rows = pRes->info.rows; @@ -4809,6 +4801,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, } initResultRowInfo(&iaInfo->binfo.resultRowInfo); + blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity); pOperator->name = "TimeMergeAlignedIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL; @@ -5076,7 +5069,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - SExecTaskInfo* pTaskInfo) { + bool mergeBlock, SExecTaskInfo* pTaskInfo) { SMergeIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (miaInfo == NULL || pOperator == NULL) { @@ -5090,6 +5083,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI iaInfo->inputOrder = TSDB_ORDER_ASC; iaInfo->interval = *pInterval; iaInfo->execModel = pTaskInfo->execModel; + iaInfo->binfo.mergeResultBlock = mergeBlock; iaInfo->primaryTsIndex = primaryTsSlotId; From 70284f5b0202317b27c05f961ee476f2ba406d8a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 16:52:36 +0800 Subject: [PATCH 07/96] enh(query): support multi-groups result merge into one datablock. --- source/libs/executor/src/timewindowoperator.c | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 812428920a..5dcfb400d9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4685,20 +4685,16 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder); } -static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { + +static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperator->info; SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } SExprSupp* pSup = &pOperator->exprSupp; SSDataBlock* pRes = iaInfo->binfo.pRes; - blockDataCleanup(pRes); - SOperatorInfo* downstream = pOperator->pDownstream[0]; int32_t scanFlag = MAIN_SCAN; @@ -4749,6 +4745,35 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { pRes->info.groupId = miaInfo->groupId; miaInfo->hasGroupId = false; +} + +static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info; + SIntervalAggOperatorInfo* iaInfo = pMiaInfo->intervalAggOperatorInfo; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSDataBlock* pRes = iaInfo->binfo.pRes; + blockDataCleanup(pRes); + + if (iaInfo->binfo.mergeResultBlock) { + while(1) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pRes->info.rows >= pOperator->resultInfo.threshold) { + break; + } + + doMergeAlignedIntervalAgg(pOperator); + } + } else { + doMergeAlignedIntervalAgg(pOperator); + } size_t rows = pRes->info.rows; pOperator->resultInfo.totalRows += rows; @@ -4812,7 +4837,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->exprSupp.numOfExprs = numOfCols; pOperator->info = miaInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeAlignedIntervalAgg, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL, destroyMergeAlignedIntervalOperatorInfo, NULL, NULL, NULL); code = appendDownstream(pOperator, &downstream, 1); From f98945e111e28061ee3c22d3d0f497262ffe9ff9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 2 Aug 2022 18:57:55 +0800 Subject: [PATCH 08/96] change log level --- source/libs/transport/src/transCli.c | 4 ++-- source/libs/transport/src/transSvr.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 431e479123..7ab3102d1c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -381,7 +381,7 @@ void cliHandleResp(SCliConn* conn) { STraceId* trace = &transMsg.info.traceId; - tGTrace("%s conn %p %s received from %s, local info:%s, len:%d, code str:%s", CONN_GET_INST_LABEL(conn), conn, + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, code str:%s", CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(pHead->msgType), conn->dst, conn->src, transMsg.contLen, tstrerror(transMsg.code)); if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { @@ -757,7 +757,7 @@ void cliSend(SCliConn* pConn) { uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); STraceId* trace = &pMsg->info.traceId; - tGTrace("%s conn %p %s is sent to %s, local info %s, len:%d", CONN_GET_INST_LABEL(pConn), pConn, + tGDebug("%s conn %p %s is sent to %s, local info %s, len:%d", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, pMsg->contLen); if (pHead->persist == 1) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 8b27d95e52..ea9d7bb910 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -253,10 +253,10 @@ static void uvHandleReq(SSvrConn* pConn) { if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); - tGTrace("%s conn %p %s received from %s, local info:%s, len:%d", transLabel(pTransInst), pConn, + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, transMsg.contLen); } else { - tGTrace("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d", transLabel(pTransInst), pConn, + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, transMsg.contLen, pHead->noResp, transMsg.code); } @@ -416,7 +416,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { STrans* pTransInst = pConn->pTransInst; STraceId* trace = &pMsg->info.traceId; - tGTrace("%s conn %p %s is sent to %s, local info:%s, len:%d", transLabel(pTransInst), pConn, + tGDebug("%s conn %p %s is sent to %s, local info:%s, len:%d", transLabel(pTransInst), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, pMsg->contLen); pHead->msgLen = htonl(len); From 77ed2bc1e13056f47c1db7f64d800eec68aa6abd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 21:25:55 +0800 Subject: [PATCH 09/96] enh(query): add merge attribute in explain. --- source/libs/command/inc/commandInt.h | 3 ++- source/libs/command/src/explain.c | 18 +++++++++++++++--- source/libs/executor/inc/executorimpl.h | 6 +++--- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/timewindowoperator.c | 5 +++-- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 2ae4666ff6..c53d464eab 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -49,6 +49,7 @@ extern "C" { #define EXPLAIN_PARITION_FORMAT "Partition on Column %s" #define EXPLAIN_ORDER_FORMAT "Order: %s" #define EXPLAIN_FILTER_FORMAT "Filter: " +#define EXPLAIN_MERGEBLOCKS_FORMAT "Merge ResBlocks: %s" #define EXPLAIN_FILL_VALUE_FORMAT "Fill Values: " #define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " #define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" @@ -56,7 +57,7 @@ extern "C" { #define EXPLAIN_TIME_WINDOWS_FORMAT "Time Window: interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c" #define EXPLAIN_WINDOW_FORMAT "Window: gap=%" PRId64 #define EXPLAIN_RATIO_TIME_FORMAT "Ratio: %f" -#define EXPLAIN_MERGE_FORMAT "Merge" +#define EXPLAIN_MERGE_FORMAT "SortMerge" #define EXPLAIN_MERGE_KEYS_FORMAT "Merge Key: " #define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms" diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index c080b666cc..2836bd7acd 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -702,7 +702,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pAggNode->mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } @@ -901,7 +905,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pIntNode->window.mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } @@ -945,7 +953,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pIntNode->window.mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d24156bc60..6383ba06b3 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -908,10 +908,10 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - SNode* pCondition, SExecTaskInfo* pTaskInfo); + SNode* pCondition, bool mergeResultBlocks, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); +SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, int32_t numOfChild); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 794e751dd2..112b6ebc99 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4131,7 +4131,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, - pPhyNode->pConditions, pTaskInfo); + pPhyNode->pConditions, pIntervalPhyNode->window.mergeDataBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) { SMergeIntervalPhysiNode* pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5dcfb400d9..aaa7a1ea8d 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4762,7 +4762,7 @@ static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) { if (iaInfo->binfo.mergeResultBlock) { while(1) { if (pOperator->status == OP_EXEC_DONE) { - return NULL; + break; } if (pRes->info.rows >= pOperator->resultInfo.threshold) { @@ -4782,7 +4782,7 @@ static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, - int32_t primaryTsSlotId, SNode* pCondition, + int32_t primaryTsSlotId, SNode* pCondition, bool mergeResultBlock, SExecTaskInfo* pTaskInfo) { SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -4806,6 +4806,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, iaInfo->interval = *pInterval; iaInfo->execModel = pTaskInfo->execModel; iaInfo->primaryTsIndex = primaryTsSlotId; + iaInfo->binfo.mergeResultBlock = mergeResultBlock; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(&pOperator->resultInfo, 4096); From 64db9afeb7c49d063d8fb6801d356ad2428212c6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 10:52:06 +0800 Subject: [PATCH 10/96] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 84 +++++++++++++++----------- 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c003f5a63f..6d1e3a4e07 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1205,6 +1205,26 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* return code; } +static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, int64_t key, SFileBlockDumpInfo* pDumpInfo) { + + // opt version + // 1. it is not a border point + // 2. the direct next point is not an duplicated timestamp + if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && pReader->order == TSDB_ORDER_ASC) || + (pDumpInfo->rowIndex > 0 && pReader->order == TSDB_ORDER_DESC)) { + int32_t step = pReader->order == TSDB_ORDER_ASC? 1:-1; + + int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; + if (nextKey != key) { // merge is not needed + doAppendRowFromBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + pDumpInfo->rowIndex += step; + return true; + } + } + + return false; +} + static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key) { SRowMerger merge = {0}; @@ -1219,10 +1239,14 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* // ascending order traverse if (ASCENDING_TRAVERSE(pReader->order)) { if (key < k.ts) { - tRowMergerInit(&merge, &fRow, pReader->pSchema); - - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); + // imem & mem are all empty, only file exist + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + } } else if (k.ts < key) { // k.ts < key doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); } else { // k.ts == key, ascending order: file block ----> imem rows -----> mem rows @@ -1238,10 +1262,13 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (key < k.ts) { doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); } else if (k.ts < key) { - tRowMergerInit(&merge, &fRow, pReader->pSchema); - - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + } } else { // descending order: mem rows -----> imem rows ------> file block updateSchema(pRow, pBlockScanInfo->uid, pReader); @@ -1438,34 +1465,23 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } // imem & mem are all empty, only file exist + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - // opt version - // 1. it is not a border point - // 2. the direct next point is not an duplicated timestamp - if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && pReader->order == TSDB_ORDER_ASC) || - (pDumpInfo->rowIndex > 0 && pReader->order == TSDB_ORDER_DESC)) { - int32_t step = pReader->order == TSDB_ORDER_ASC? 1:-1; - int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; - if (nextKey != key) { // merge is not needed - doAppendRowFromBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); - pDumpInfo->rowIndex += step; - return TSDB_CODE_SUCCESS; - } + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; } - - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - - STSRow* pTSRow = NULL; - SRowMerger merge = {0}; - - tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); - - taosMemoryFree(pTSRow); - tRowMergerClear(&merge); - return TSDB_CODE_SUCCESS; } } From bc600a4942e5a82cad3d2749f613838f9315b33a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 14:08:03 +0800 Subject: [PATCH 11/96] refactor: do some internal refactor optimize the building block performance. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 76 +++++++++++++++++++------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6d1e3a4e07..62d7a72b8f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -152,7 +152,7 @@ static void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); static void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, - STsdbReader* pReader); + STsdbReader* pReader, bool* freeTSRow); static void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow); static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData, @@ -1235,6 +1235,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* TSDBKEY k = TSDBROW_KEY(pRow); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SArray* pDelList = pBlockScanInfo->delSkyline; + bool freeTSRow = false; // ascending order traverse if (ASCENDING_TRAVERSE(pReader->order)) { @@ -1248,7 +1249,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerGetRow(&merge, &pTSRow); } } else if (k.ts < key) { // k.ts < key - doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader, &freeTSRow); } else { // k.ts == key, ascending order: file block ----> imem rows -----> mem rows tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); @@ -1260,7 +1261,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } else { // descending order scan if (key < k.ts) { - doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader, &freeTSRow); } else if (k.ts < key) { if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { return TSDB_CODE_SUCCESS; @@ -1302,12 +1303,12 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* ASSERT(pRow != NULL && piRow != NULL); int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + bool freeTSRow = false; uint64_t uid = pBlockScanInfo->uid; TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); - if (ASCENDING_TRAVERSE(pReader->order)) { // [1&2] key <= [k.ts && ik.ts] if (key <= k.ts && key <= ik.ts) { @@ -1335,16 +1336,22 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [3] ik.ts < key <= k.ts // [4] ik.ts < k.ts <= key if (ik.ts < k.ts) { - doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader); + doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } return TSDB_CODE_SUCCESS; } // [5] k.ts < key <= ik.ts // [6] k.ts < ik.ts <= key if (k.ts < ik.ts) { - doMergeMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader); + doMergeMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader, &freeTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } return TSDB_CODE_SUCCESS; } @@ -1354,6 +1361,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } } @@ -1385,8 +1393,11 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [3] ik.ts > k.ts >= Key // [4] ik.ts > key >= k.ts if (ik.ts > key) { - doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader); + doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } return TSDB_CODE_SUCCESS; } @@ -1399,18 +1410,21 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } //[7] key = ik.ts > k.ts if (key == ik.ts) { - doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader); + doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); tRowMerge(&merge, &fRow); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + + taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } } @@ -2280,11 +2294,29 @@ void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader) { } void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, - STsdbReader* pReader) { - SRowMerger merge = {0}; + STsdbReader* pReader, bool* freeTSRow) { + { // if the timestamp of the next valid row has a different ts, return current row directly + TSDBROW current = *pRow; + pIter->hasVal = tsdbTbDataIterNext(pIter->iter); - TSDBKEY k = TSDBROW_KEY(pRow); - // updateSchema(pRow, uid, pReader); + if (!pIter->hasVal) { + *pTSRow = current.pTSRow; + *freeTSRow = false; + return; + } else { // has next point in mem/imem + TSDBROW* pNextRow = getValidRow(pIter, pDelList, pReader); + if (TSDBROW_KEY(¤t).ts != TSDBROW_KEY(pNextRow).ts) { + *pTSRow = current.pTSRow; + *freeTSRow = false; + return; + } + } + } + + SRowMerger merge = {0}; + TSDBKEY k = TSDBROW_KEY(pRow); + + // get the correct schema for data in memory int32_t sversion = TSDBROW_SVERSION(pRow); STSchema* pTSchema = NULL; if (sversion != pReader->pSchema->version) { @@ -2298,6 +2330,8 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe tRowMergerGetRow(&merge, pTSRow); tRowMergerClear(&merge); + *freeTSRow = true; + if (sversion != pReader->pSchema->version) { taosMemoryFree(pTSchema); } @@ -2332,7 +2366,7 @@ void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlo } int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow, - int64_t endKey) { + int64_t endKey, bool* freeTSRow) { TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); SArray* pDelList = pBlockScanInfo->delSkyline; @@ -2358,23 +2392,24 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR TSDBKEY ik = TSDBROW_KEY(piRow); if (ik.ts < k.ts) { // ik.ts < k.ts - doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader); + doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); } else if (k.ts < ik.ts) { - doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); } else { // ik.ts == k.ts doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, pTSRow); + *freeTSRow = true; } return TSDB_CODE_SUCCESS; } if (pBlockScanInfo->iter.hasVal && pRow != NULL) { - doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); return TSDB_CODE_SUCCESS; } if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { - doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader); + doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); return TSDB_CODE_SUCCESS; } @@ -2472,13 +2507,16 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e do { STSRow* pTSRow = NULL; - tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow, endKey); + bool freeTSRow = false; + tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow, endKey, &freeTSRow); if (pTSRow == NULL) { break; } doAppendRowFromTSRow(pBlock, pReader, pTSRow); - taosMemoryFree(pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } // no data in buffer, return immediately if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) { From 074500805c1745035d50217758115067360db5ae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 15:34:42 +0800 Subject: [PATCH 12/96] refactor: set filter function for last_row function. --- source/libs/executor/src/scanoperator.c | 5 ++--- source/libs/function/src/builtins.c | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 43210ced2a..b0f05e4705 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -521,7 +521,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; - qDebug("%s start to repeat ascending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); + qDebug("%s start to repeat ascending order scan data SELECT last_row(*),hostname from cpu group by hostname;blocks due to query func required", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); @@ -533,10 +533,9 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(pTableScanInfo, pOperator->exprSupp.pCtx, 0); tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); + qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); } - qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); - while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 23e26f7716..444cd2c5d0 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2271,6 +2271,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_LAST_ROW, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLast, + .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastRowFunction, From a3166507a54baf86a31d45f91935deecc9911229 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 15:58:28 +0800 Subject: [PATCH 13/96] enh: don't read bitmap when all col is normal --- include/common/trow.h | 8 ++-- source/client/src/tmq.c | 2 + source/common/src/tdatablock.c | 1 + source/common/src/trow.c | 56 +++++++++++++++---------- source/dnode/vnode/src/tq/tqSink.c | 1 + source/libs/executor/src/dataInserter.c | 1 + source/libs/parser/src/parInsert.c | 7 +++- source/libs/parser/src/parInsertData.c | 1 + 8 files changed, 52 insertions(+), 25 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 807a4c0f0a..93004b2cbd 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -38,7 +38,8 @@ typedef struct { uint16_t type : 2; uint16_t del : 1; uint16_t endian : 1; - uint16_t reserve : 12; + uint16_t normal : 1; // all norm + uint16_t reserve : 11; uint16_t sver; }; }; @@ -149,6 +150,8 @@ typedef struct { void *pBitmap; void *pOffset; int32_t extendedRowSize; + int16_t nNone; + int16_t nNull; } SRowBuilder; #define TD_ROW_HEAD_LEN (sizeof(STSRow)) @@ -287,9 +290,8 @@ int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen); int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, int32_t allNullLen, int32_t boundNullLen); int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); +int32_t tdSRowEnd(SRowBuilder *pBuilder); int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf); -int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, - int32_t nBoundCols, int32_t flen); void tdSRowReset(SRowBuilder *pBuilder); int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index bdd8c75f26..46349a3327 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -3129,6 +3129,7 @@ int taos_write_raw_block(TAOS *taos, int rows, char *pData, const char* tbname){ offset += TYPE_BYTES[pColumn->type]; } + tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); rowData = POINTER_SHIFT(rowData, rowLen); dataLen += rowLen; @@ -3360,6 +3361,7 @@ static int32_t tmqWriteRaw(TAOS *taos, void* data, int32_t dataLen){ } offset += TYPE_BYTES[pColumn->type]; } + tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); rowData = POINTER_SHIFT(rowData, rowLen); dataLen += rowLen; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 613fdd6b77..a16364b4a9 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1906,6 +1906,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks } offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation } + tdSRowEnd(&rb); dataLen += TD_ROW_LEN(rb.pBuf); #ifdef TD_DEBUG_PRINT_ROW tdSRowPrint(rb.pBuf, pTSchema, __func__); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 754e142437..6558af8aad 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -666,6 +666,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { ++iColVal; } + tdSRowEnd(&rb); taosMemoryFreeClear(varBuf); @@ -880,26 +881,20 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, int16_t colIdx) { -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { - output->valType = TD_VTYPE_NONE; - return terrno; - } - if (tdValTypeIsNorm(output->valType)) { + if (pRow->normal) { if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); } + return TSDB_CODE_SUCCESS; } -#else - if (IS_VAR_DATA_TYPE(colType)) { - output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + + if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { + output->valType = TD_VTYPE_NONE; + return terrno; } - output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif + return TSDB_CODE_SUCCESS; } @@ -925,6 +920,21 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp return TSDB_CODE_SUCCESS; } // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. + + switch (valType) { + case TD_VTYPE_NORM: + break; + case TD_VTYPE_NULL: + ++pBuilder->nNull; + break; + case TD_VTYPE_NONE: + ++pBuilder->nNone; + break; + default: + ASSERT(0); + break; + } + if (TD_IS_TP_ROW(pRow)) { tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset); } else { @@ -1084,6 +1094,14 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou return TSDB_CODE_SUCCESS; } +int32_t tdSRowEnd(SRowBuilder *pBuilder) { + STSRow *pRow = (STSRow *)pBuilder->pBuf; + if (pBuilder->nNone || pBuilder->nNull) { + pRow->normal = 1; + } + return TSDB_CODE_SUCCESS; +} + int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { @@ -1092,6 +1110,10 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { return terrno; } + if (pBuilder->nNone) pBuilder->nNone = 0; + if (pBuilder->nNull) pBuilder->nNull = 0; + + pBuilder->nNull = 0; TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); @@ -1159,14 +1181,6 @@ int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { return TSDB_CODE_SUCCESS; } -int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, - int32_t nBoundCols, int32_t flen) { - if (tdSRowSetExtendedInfo(pBuilder, allNullLen, boundNullLen, nCols, nBoundCols, flen) < 0) { - return terrno; - } - return tdSRowResetBuf(pBuilder, pBuf); -} - void tdSRowReset(SRowBuilder *pBuilder) { pBuilder->rowType = TD_ROW_TP; pBuilder->pBuf = NULL; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 850394b15d..fc6f2b0fa8 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -157,6 +157,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); } } + tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); rowData = POINTER_SHIFT(rowData, rowLen); dataLen += rowLen; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 1c08fafaa3..5e43e52afa 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -213,6 +213,7 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); } } + tdSRowEnd(&rb); if (ignoreRow) { continue; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index fbc5e1b4b0..d1cf82ff1f 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1206,6 +1206,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, } // *len = pBuilder->extendedRowSize; + tdSRowEnd(pBuilder); return TSDB_CODE_SUCCESS; } @@ -1973,6 +1974,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in } } } + tdSRowEnd(pBuilder); #ifdef TD_DEBUG_PRINT_ROW STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); tdSRowPrint(row, pSTSchema, __func__); @@ -2056,7 +2058,9 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu } } } - + if(rowEnd) { + tdSRowEnd(pBuilder); + } #ifdef TD_DEBUG_PRINT_ROW if (rowEnd) { STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); @@ -2436,6 +2440,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols } } + tdSRowEnd(pBuilder); pDataBlock->size += extendedRowSize; } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index ae123a3563..ee35b23cc0 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -659,6 +659,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB toffset += TYPE_BYTES[colType]; p += pSchema[j].bytes; } + tdSRowEnd(&builder); int32_t rowLen = TD_ROW_LEN((STSRow*)pDataBlock); pDataBlock = (char*)pDataBlock + rowLen; pBlock->dataLen += rowLen; From 0c4454c07764d99682d3d4d6992ba8357ef98427 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:23:27 +0800 Subject: [PATCH 14/96] fix: set statis to 0 when all col is normal --- include/common/trow.h | 2 +- source/common/src/trow.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 93004b2cbd..f2f68ddccd 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -38,7 +38,7 @@ typedef struct { uint16_t type : 2; uint16_t del : 1; uint16_t endian : 1; - uint16_t normal : 1; // all norm + uint16_t statis : 1; // 0 all normal, 1 has null or none uint16_t reserve : 11; uint16_t sver; }; diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6558af8aad..b54d653df9 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -881,7 +881,7 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, int16_t colIdx) { - if (pRow->normal) { + if (pRow->statis == 0) { if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { @@ -1097,7 +1097,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou int32_t tdSRowEnd(SRowBuilder *pBuilder) { STSRow *pRow = (STSRow *)pBuilder->pBuf; if (pBuilder->nNone || pBuilder->nNull) { - pRow->normal = 1; + pRow->statis = 1; } return TSDB_CODE_SUCCESS; } From 2b06d8328c711fdb537ac2bff21310b426311afb Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:27:50 +0800 Subject: [PATCH 15/96] enh: row end optimization --- include/common/trow.h | 7 ++++++- source/common/src/trow.c | 8 -------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index f2f68ddccd..35270341ff 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -290,7 +290,12 @@ int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen); int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, int32_t allNullLen, int32_t boundNullLen); int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); -int32_t tdSRowEnd(SRowBuilder *pBuilder); +static FORCE_INLINE void tdSRowEnd(SRowBuilder *pBuilder) { + STSRow *pRow = (STSRow *)pBuilder->pBuf; + if (pBuilder->nNone || pBuilder->nNull) { + pRow->statis = 1; + } +} int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf); void tdSRowReset(SRowBuilder *pBuilder); int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, diff --git a/source/common/src/trow.c b/source/common/src/trow.c index b54d653df9..ec51d65cdd 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -1094,14 +1094,6 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou return TSDB_CODE_SUCCESS; } -int32_t tdSRowEnd(SRowBuilder *pBuilder) { - STSRow *pRow = (STSRow *)pBuilder->pBuf; - if (pBuilder->nNone || pBuilder->nNull) { - pRow->statis = 1; - } - return TSDB_CODE_SUCCESS; -} - int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { From 22512c09222c82055038d92fd5e3581ba1fd1f78 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:33:53 +0800 Subject: [PATCH 16/96] enh: row optimization --- include/common/trow.h | 6 +++--- source/common/src/trow.c | 9 ++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 35270341ff..bd8510dde6 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -150,8 +150,8 @@ typedef struct { void *pBitmap; void *pOffset; int32_t extendedRowSize; - int16_t nNone; - int16_t nNull; + bool hasNone; + bool hasNull; } SRowBuilder; #define TD_ROW_HEAD_LEN (sizeof(STSRow)) @@ -292,7 +292,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); static FORCE_INLINE void tdSRowEnd(SRowBuilder *pBuilder) { STSRow *pRow = (STSRow *)pBuilder->pBuf; - if (pBuilder->nNone || pBuilder->nNull) { + if (pBuilder->hasNull || pBuilder->hasNone) { pRow->statis = 1; } } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index ec51d65cdd..d968221432 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -925,10 +925,10 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp case TD_VTYPE_NORM: break; case TD_VTYPE_NULL: - ++pBuilder->nNull; + if (!pBuilder->hasNull) pBuilder->hasNull = true; break; case TD_VTYPE_NONE: - ++pBuilder->nNone; + if (!pBuilder->hasNone) pBuilder->hasNone = true; break; default: ASSERT(0); @@ -1102,10 +1102,9 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { return terrno; } - if (pBuilder->nNone) pBuilder->nNone = 0; - if (pBuilder->nNull) pBuilder->nNull = 0; + if (pBuilder->hasNone) pBuilder->hasNone = false; + if (pBuilder->hasNull) pBuilder->hasNull = false; - pBuilder->nNull = 0; TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); From 258448bae9ca9b40775d47dd22fb76d7f1f0c320 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:42:08 +0800 Subject: [PATCH 17/96] fix: get normal val from tp row --- source/common/src/trow.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d968221432..a096db4b1d 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -882,6 +882,7 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, int16_t colIdx) { if (pRow->statis == 0) { + output->valType = TD_VTYPE_NORM; if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { @@ -895,6 +896,14 @@ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t return terrno; } + if (tdValTypeIsNorm(output->valType)) { + if (IS_VAR_DATA_TYPE(colType)) { + output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + } + return TSDB_CODE_SUCCESS; } From 843a516919eb5aa37add33e3cee09b9e8405e700 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:44:23 +0800 Subject: [PATCH 18/96] enh: get normal val from tp row --- source/common/src/trow.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index a096db4b1d..6e6b5acecf 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -896,7 +896,7 @@ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t return terrno; } - if (tdValTypeIsNorm(output->valType)) { + if (output->valType == TD_VTYPE_NORM) { if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { From 846ee63c88f8eb52cc6f47dd9bd01d5c3ce90f81 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:47:41 +0800 Subject: [PATCH 19/96] enh: normal val optimization --- source/common/src/trow.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6e6b5acecf..6f58a0ded1 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -912,7 +912,7 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp STSRow *pRow = pBuilder->pBuf; if (!val) { #ifdef TD_SUPPORT_BITMAP - if (tdValTypeIsNorm(valType)) { + if (valType == TD_VTYPE_NORM) { terrno = TSDB_CODE_INVALID_PTR; return terrno; } @@ -970,7 +970,7 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo STSRow *row = pBuilder->pBuf; // No need to store None/Null values. - if (tdValIsNorm(valType, val, colType)) { + if (valType == TD_VTYPE_NORM) { // ts key stored in STSRow.ts SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); @@ -1031,7 +1031,7 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo // 1. No need to set flen part for Null/None, just use bitmap. When upsert for the same primary TS key, the bitmap // should be updated simultaneously if Norm val overwrite Null/None cols. // 2. When consume STSRow in memory by taos client/tq, the output of Null/None cols should both be Null. - if (tdValIsNorm(valType, val, colType)) { + if (valType == TD_VTYPE_NORM) { // TODO: The layout of new data types imported since 3.0 like blob/medium blob is the same with binary/nchar. if (IS_VAR_DATA_TYPE(colType)) { // ts key stored in STSRow.ts From c8c9bbdeb1a6f6fa948a287be6ba3a4ec08a9222 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 3 Aug 2022 17:04:14 +0800 Subject: [PATCH 20/96] enh: last_row function scan in reverse order --- source/libs/planner/src/planOptimizer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index c02ef64707..ddf8b29153 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -242,7 +242,7 @@ static int32_t scanPathOptGetScanOrder(SScanLogicNode* pScan, EScanOrder* pScanO SFunctionNode* pFunc = (SFunctionNode*)pNode; if (FUNCTION_TYPE_FIRST == pFunc->funcType) { hasFirst = true; - } else if (FUNCTION_TYPE_LAST == pFunc->funcType) { + } else if (FUNCTION_TYPE_LAST == pFunc->funcType || FUNCTION_TYPE_LAST_ROW == pFunc->funcType) { hasLast = true; } else if (FUNCTION_TYPE_SELECT_VALUE != pFunc->funcType) { otherFunc = true; @@ -436,8 +436,8 @@ static int32_t pushDownCondOptDealScan(SOptimizeContext* pCxt, SScanLogicNode* p SNode* pPrimaryKeyCond = NULL; SNode* pOtherCond = NULL; - int32_t code = filterPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, &pScan->pTagCond, - &pOtherCond); + int32_t code = filterPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, + &pScan->pTagCond, &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pScan->pTagCond) { code = pushDownCondOptRebuildTbanme(&pScan->pTagCond); } From caa5378d8e2f2f88d64f00f53bf8ddbb711fdedd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 17:12:10 +0800 Subject: [PATCH 21/96] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b0f05e4705..93a842d24c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -296,7 +296,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; From 2a8da022def0cad9a04569ef44b72e60887a760b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 19:01:06 +0800 Subject: [PATCH 22/96] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 3 --- source/libs/function/src/builtins.c | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 93a842d24c..a6addc208d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -675,9 +675,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; -// pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose -// pInfo->cond.order = TSDB_ORDER_DESC; - pInfo->pdInfo.interval = extractIntervalInfo(pTableScanNode); pInfo->readHandle = *readHandle; pInfo->sample.sampleRatio = pTableScanNode->ratio; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 444cd2c5d0..1c519bae48 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2276,6 +2276,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = lastRowFunction, .sprocessFunc = firstLastScalarFunction, + .pPartialFunc = "_last_row_partial", + .pMergeFunc = "_last_row_merge", .finalizeFunc = firstLastFinalize }, { @@ -2288,6 +2290,27 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = cachedLastRowFunction, .finalizeFunc = firstLastFinalize }, + { + .name = "_last_row_partial", + .type = FUNCTION_TYPE_LAST_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .translateFunc = translateFirstLastPartial, + .dynDataRequiredFunc = lastDynDataReq, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastRowFunction, + .finalizeFunc = firstLastPartialFinalize, + }, + { + .name = "_last_row_merge", + .type = FUNCTION_TYPE_LAST_MERGE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .translateFunc = translateFirstLastMerge, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunctionMerge, + .finalizeFunc = firstLastFinalize, + }, { .name = "first", .type = FUNCTION_TYPE_FIRST, From c253ea73e1a6dd62870569465017ba6a77f5d8f1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 19:17:07 +0800 Subject: [PATCH 23/96] fix(query): fix the null not returns problem. --- source/libs/function/src/builtinsimpl.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 38c35e382a..a4e74e7a7d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2953,25 +2953,24 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) { SInputColumnInfoData* pColInfo = &pCtx->input; - int32_t start = pColInfo->startRowIndex; - - pOutput->bytes = pInput->bytes; - TSKEY* tsIn = &pInput->ts; - TSKEY* tsOut = &pOutput->ts; + int32_t start = pColInfo->startRowIndex; if (pOutput->hasResult) { if (isFirst) { - if (*tsIn > *tsOut) { + if (pInput->ts > pOutput->ts) { return; } } else { - if (*tsIn < *tsOut) { + if (pInput->ts < pOutput->ts) { return; } } } - *tsOut = *tsIn; + pOutput->isNull = pInput->isNull; + pOutput->ts = pInput->ts; + pOutput->bytes = pInput->bytes; + memcpy(pOutput->buf, pInput->buf, pOutput->bytes); saveTupleData(pCtx->pSrcBlock, start, pCtx, pOutput); @@ -2998,7 +2997,6 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer } SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); - return TSDB_CODE_SUCCESS; } @@ -3099,9 +3097,9 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { TSKEY startKey = getRowPTs(pInput->pPTS, 0); TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); +#if 0 int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; -#if 0 // the optimized version only function if all tuples in one block are monotonious increasing or descreasing. // this is NOT always works if project operator exists in downstream. if (blockDataOrder == TSDB_ORDER_ASC) { @@ -3141,6 +3139,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { } #endif + SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } From 0410959aacdae6b495fae1b547ecd47ee14ad9f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 19:19:17 +0800 Subject: [PATCH 24/96] test: disable some unit case. --- source/client/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 95e86e7629..f52edbe71f 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -46,6 +46,6 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) -#if(${BUILD_TEST}) +if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -#endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) \ No newline at end of file From 3566e8cf46913e7acc0e8be9266100d036e35096 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 15:02:05 +0800 Subject: [PATCH 25/96] enh(query): diff support timestamp column type. --- source/libs/function/src/builtins.c | 4 +--- source/libs/function/src/builtinsimpl.c | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1c519bae48..4ef2525b36 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1523,7 +1523,7 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType) { + if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType && TSDB_DATA_TYPE_TIMESTAMP != colType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -2204,8 +2204,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = elapsedFinalize, .invertFunc = NULL, .combineFunc = elapsedCombine, - //.pPartialFunc = "_elapsed_partial", - //.pMergeFunc = "_elapsed_merge" }, { .name = "_elapsed_partial", diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a4e74e7a7d..650aa3f12d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3170,6 +3170,8 @@ bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { switch (type) { case TSDB_DATA_TYPE_BOOL: + pDiffInfo->prev.i64 = *(bool*)pv? 1:0; + break; case TSDB_DATA_TYPE_TINYINT: pDiffInfo->prev.i64 = *(int8_t*)pv; break; @@ -3179,6 +3181,7 @@ static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { case TSDB_DATA_TYPE_SMALLINT: pDiffInfo->prev.i64 = *(int16_t*)pv; break; + case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: pDiffInfo->prev.i64 = *(int64_t*)pv; break; @@ -3232,6 +3235,7 @@ static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SCo pDiffInfo->prev.i64 = v; break; } + case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: { int64_t v = *(int64_t*)pv; int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null From 271528f4e21249c3e424a49d62634c1aed67d140 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 18:12:44 +0800 Subject: [PATCH 26/96] refactor: opt tsdb performance. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9b5523431c..7a867d85c9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2113,7 +2113,7 @@ TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pRea } TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); - TSDBKEY key = TSDBROW_KEY(pRow); + TSDBKEY key = {.ts = pRow->pTSRow->ts, .version = pRow->version}; if (outOfTimeWindow(key.ts, &pReader->window)) { pIter->hasVal = false; return NULL; From be1d1c5f8ecf6607151d8a539dcb9058e913b74f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 20:37:36 +0800 Subject: [PATCH 27/96] fix(query): clean temp files. --- source/libs/executor/src/groupoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 938134c167..528b39c17f 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -727,7 +727,7 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFree(pInfo->columnOffset); cleanupExprSupp(&pInfo->scalarSup); - + clearDiskbasedBuf(pInfo->pBuf); taosMemoryFreeClear(param); } From 27d223356a1df0a11a0c2d56801505e5c4923c3b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 4 Aug 2022 20:41:30 +0800 Subject: [PATCH 28/96] enh: merge multiple blocks in one rsp msg --- include/common/tmsg.h | 1 + include/libs/executor/dataSinkMgt.h | 1 + source/libs/qworker/inc/qwInt.h | 2 + source/libs/qworker/src/qwMsg.c | 9 ++- source/libs/qworker/src/qworker.c | 114 ++++++++++++++++++---------- 5 files changed, 83 insertions(+), 44 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bfb80ec8f8..9d001c9534 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1364,6 +1364,7 @@ typedef struct { int8_t compressed; int8_t streamBlockType; int32_t compLen; + int32_t numOfBlocks; int32_t numOfRows; int32_t numOfCols; int64_t skey; diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 47177dc11b..71105d88eb 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -67,6 +67,7 @@ typedef struct SInputData { } SInputData; typedef struct SOutputData { + int32_t numOfBlocks; int32_t numOfRows; int32_t numOfCols; int8_t compressed; diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 8f036714c9..729ac474e4 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -35,6 +35,7 @@ extern "C" { #define QW_DEFAULT_SHORT_RUN_TIMES 2 #define QW_DEFAULT_HEARTBEAT_MSEC 5000 #define QW_SCH_TIMEOUT_MSEC 180000 +#define QW_MIN_RES_ROWS 4096 enum { QW_PHASE_PRE_QUERY = 1, @@ -135,6 +136,7 @@ typedef struct SQWTaskCtx { int32_t msgType; int32_t fetchType; int32_t execId; + int32_t level; bool queryRsped; bool queryEnd; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index e8ffd98153..63a3c1bfea 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -12,14 +12,16 @@ int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { int32_t msgSize = sizeof(SRetrieveTableRsp) + length; - SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(msgSize); + SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcReallocCont(*rsp, msgSize); if (NULL == pRsp) { qError("rpcMallocCont %d failed", msgSize); QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - memset(pRsp, 0, sizeof(SRetrieveTableRsp)); - + if (NULL == *rsp) { + memset(pRsp, 0, sizeof(SRetrieveTableRsp)); + } + *rsp = pRsp; return TSDB_CODE_SUCCESS; @@ -35,6 +37,7 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) rsp->compLen = htonl(len); rsp->numOfRows = htonl(input->numOfRows); rsp->numOfCols = htonl(input->numOfCols); + rsp->numOfBlocks = htonl(input->numOfBlocks); } void qwFreeFetchRsp(void *msg) { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 36d85f1f12..d93c07ce1e 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -203,57 +203,88 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, SRetrieveTableRsp *rsp = NULL; bool queryEnd = false; int32_t code = 0; + SOutputData output = {0}; - dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); + *dataLen = 0; - if (len < 0) { - QW_TASK_ELOG("invalid length from dsGetDataLength, length:%d", len); - QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + while (true) { + dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); - if (len == 0) { - if (queryEnd) { - code = dsGetDataBlock(ctx->sinkHandle, pOutput); - if (code) { - QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); - } - - QW_TASK_DLOG_E("no data in sink and query end"); - - qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); - QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); - - *rspMsg = rsp; - *dataLen = 0; - return TSDB_CODE_SUCCESS; + if (len < 0) { + QW_TASK_ELOG("invalid length from dsGetDataLength, length:%d", len); + QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - pOutput->bufStatus = DS_BUF_EMPTY; + if (len == 0) { + if (queryEnd) { + code = dsGetDataBlock(ctx->sinkHandle, &output); + if (code) { + QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); + } - return TSDB_CODE_SUCCESS; + QW_TASK_DLOG("no more data in sink and query end, fetched blocks %d rows %d", pOutput->numOfBlocks, pOutput->numOfRows); + + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); + if (NULL == rsp) { + QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); + *pOutput = output; + } else { + pOutput->queryEnd = output.queryEnd; + pOutput->bufStatus = output.bufStatus; + pOutput->useconds = output.useconds; + } + + break; + } + + pOutput->bufStatus = DS_BUF_EMPTY; + + break; + } + + // Got data from sink + QW_TASK_DLOG("there are data in sink, dataLength:%d", len); + + *dataLen += len; + + QW_ERR_RET(qwMallocFetchRsp(*dataLen, &rsp)); + + output.pData = rsp->data + *dataLen - len; + code = dsGetDataBlock(ctx->sinkHandle, &output); + if (code) { + QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); + } + + pOutput->queryEnd = output.queryEnd; + pOutput->precision = output.precision; + pOutput->bufStatus = output.bufStatus; + pOutput->useconds = output.useconds; + pOutput->compressed = output.compressed; + pOutput->numOfCols = output.numOfCols; + pOutput->numOfRows += output.numOfRows; + pOutput->numOfBlocks++; + + if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { + QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %d", pOutput->numOfBlocks, pOutput->numOfRows); + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); + break; + } + + if (0 == ctx->level) { + QW_TASK_DLOG("task fetched blocks %d rows %d, level %d", pOutput->numOfBlocks, pOutput->numOfRows, ctx->level); + break; + } + + if (pOutput->numOfRows >= QW_MIN_RES_ROWS) { + QW_TASK_DLOG("task fetched blocks %d rows %d reaches the min rows", pOutput->numOfBlocks, pOutput->numOfRows); + break; + } } - // Got data from sink - QW_TASK_DLOG("there are data in sink, dataLength:%d", len); - - *dataLen = len; - - QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); *rspMsg = rsp; - pOutput->pData = rsp->data; - code = dsGetDataBlock(ctx->sinkHandle, pOutput); - if (code) { - QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); - } - - if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { - QW_TASK_DLOG_E("task all data fetched, done"); - qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); - } - return TSDB_CODE_SUCCESS; } @@ -551,6 +582,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) { // queryRsped = true; + ctx->level = plan->level; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); atomic_store_ptr(&ctx->sinkHandle, sinkHandle); From 569c82886f4d26488a896b43e785004b64202b56 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 20:42:23 +0800 Subject: [PATCH 29/96] fix(query): remove temp files. --- source/libs/executor/src/groupoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 528b39c17f..507719e0aa 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -727,7 +727,7 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFree(pInfo->columnOffset); cleanupExprSupp(&pInfo->scalarSup); - clearDiskbasedBuf(pInfo->pBuf); + destroyDiskbasedBuf(pInfo->pBuf); taosMemoryFreeClear(param); } From 0e4748771acd90aceeee5141b094bbce64e7d1ed Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 14:10:45 +0800 Subject: [PATCH 30/96] enh(query): support combine multi datablock to pass to upstream operators --- include/common/tdatablock.h | 2 +- source/client/src/clientImpl.c | 11 ++ source/common/src/tdatablock.c | 34 +++++- source/dnode/vnode/src/tsdb/tsdbRead.c | 7 +- source/libs/executor/inc/executorimpl.h | 9 +- source/libs/executor/src/dataDispatcher.c | 8 +- source/libs/executor/src/executorimpl.c | 142 +++++++++++++--------- source/libs/executor/src/scanoperator.c | 4 +- source/libs/stream/src/streamData.c | 5 +- 9 files changed, 142 insertions(+), 80 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 7839859e8b..3d605a144b 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -239,7 +239,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); -const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); +const char* blockDecode(SSDataBlock* pBlock, const char* pData); void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag); void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0c4dc1705c..8357e2d627 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1765,6 +1765,17 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t dataLen = *(int32_t*)p; p += sizeof(int32_t); + int32_t rows = *(int32_t*)p; + p += sizeof(int32_t); + + int32_t cols = *(int32_t*)p; + p += sizeof(int32_t); + + ASSERT(rows == numOfRows && cols == numOfCols); + + int32_t hasColumnSeg = *(int32_t*)p; + p += sizeof(int32_t); + uint64_t groupId = *(uint64_t*)p; p += sizeof(uint64_t); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4ce60efaea..bcbdf8d786 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -676,9 +676,9 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { * @return */ size_t blockDataGetSerialMetaSize(uint32_t numOfCols) { - // | total rows/total length | block group id | column schema | each column length | - return sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + - numOfCols * sizeof(int32_t); + // | total length | total rows | total columns | has column seg| block group id | column schema | each column length | + return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); } double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { @@ -2077,6 +2077,18 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ int32_t* actualLen = (int32_t*)data; data += sizeof(int32_t); + int32_t* rows = (int32_t*)data; + *rows = pBlock->info.rows; + data += sizeof(int32_t); + + int32_t* cols = (int32_t*)data; + *cols = numOfCols; + data += sizeof(int32_t); + + int32_t* hasColumnSegment = (int32_t*)data; + *hasColumnSegment = 1; + data += sizeof(int32_t); + uint64_t* groupId = (uint64_t*)data; data += sizeof(uint64_t); @@ -2130,12 +2142,26 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ *groupId = pBlock->info.groupId; } -const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { +const char* blockDecode(SSDataBlock* pBlock, const char* pData) { const char* pStart = pData; + // total length sizeof(int32_t) int32_t dataLen = *(int32_t*)pStart; pStart += sizeof(int32_t); + // total rows sizeof(int32_t) + int32_t numOfRows = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + // total columns sizeof(int32_t) + int32_t numOfCols = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + // has column info segment + int32_t hasColumnInfo = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + // group id sizeof(uint64_t) pBlock->info.groupId = *(uint64_t*)pStart; pStart += sizeof(uint64_t); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7a867d85c9..2a8e234ac0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2714,7 +2714,6 @@ void tsdbReaderClose(STsdbReader* pReader) { } SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap); taosMemoryFreeClear(pSupInfo->plist); @@ -2742,10 +2741,8 @@ void tsdbReaderClose(STsdbReader* pReader) { SIOCostSummary* pCost = &pReader->cost; tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 - " SMA-time:%.2f ms, " - "fileBlocks:%" PRId64 - ", fileBlocks-time:%.2f ms, build in-memory-block-time:%.2f ms, STableBlockScanInfo " - "size:%.2f Kb %s", + " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-time:%.2f ms, " + "build in-memory-block-time:%.2f ms, STableBlockScanInfo size:%.2f Kb %s", pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaData, pCost->smaLoadTime, pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 2e0ed802e5..62b2cc0fa6 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -266,7 +266,11 @@ typedef struct SExchangeInfo { SArray* pSourceDataInfo; tsem_t ready; void* pTransporter; - SSDataBlock* pResult; + // SArray, result block list, used to keep the multi-block that + // passed by downstream operator + SArray* pResultBlockList; + int32_t rspBlockIndex; // indicate the return block index in pResultBlockList + SSDataBlock* pDummyBlock; // dummy block, not keep data bool seqLoadData; // sequential load data or not, false by default int32_t current; SLoadRemoteDataInfo loadInfo; @@ -855,8 +859,7 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWin int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, - SArray* pColList); + int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart); STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index f0fb5852a0..3a37b5e760 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -69,10 +69,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { // clang-format off // data format: -// +----------------+--------------+-----------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) -// | |sizeof(int32) |sizeof(uint64_t) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | -// +----------------+--------------+-----------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// |SDataCacheEntry | total length | numOfRows | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) +// | |sizeof(int32) | sizeof(int32)| sizeof(uint64_t) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | +// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header // clang-format on diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 23dbd4d992..8dfda896cc 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1956,6 +1956,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->compLen = htonl(pRsp->compLen); pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); + pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); qDebug("%s fetch rsp received, index:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfRows); @@ -2043,12 +2044,10 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf } int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, - SArray* pColList) { + int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart) { if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); - // blockDataEnsureCapacity(pRes, numOfRows); - blockDecode(pRes, numOfOutput, numOfRows, pData); + *pNextStart = (char*) blockDecode(pRes, pData); } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; @@ -2072,7 +2071,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo blockDataAppendColInfo(pBlock, &idata); } - blockDecode(pBlock, numOfCols, numOfRows, pStart); + blockDecode(pBlock, pStart); blockDataEnsureCapacity(pRes, numOfRows); // data from mnode @@ -2084,8 +2083,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. blockDataUpdateTsWindow(pRes, 0); - int64_t el = taosGetTimestampUs() - startTs; - pLoadInfo->totalRows += numOfRows; pLoadInfo->totalSize += compLen; @@ -2093,7 +2090,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo *total += numOfRows; } - pLoadInfo->totalElapsed += el; return TSDB_CODE_SUCCESS; } @@ -2115,8 +2111,8 @@ static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { return NULL; } -static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, - SExecTaskInfo* pTaskInfo) { +static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, + SExecTaskInfo* pTaskInfo) { int32_t code = 0; int64_t startTs = taosGetTimestampUs(); size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); @@ -2142,7 +2138,6 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx SRetrieveTableRsp* pRsp = pDataInfo->pRsp; SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); - SSDataBlock* pRes = pExchangeInfo->pResult; SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pRsp->numOfRows == 0) { qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 @@ -2155,29 +2150,37 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx continue; } - SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; - code = - extractDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); - if (code != 0) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + int32_t index = 0; + char* pStart = pRetrieveRsp->data; + while(index++ < pRetrieveRsp->numOfBlocks) { + SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); + blockDataEnsureCapacity(pb, pRetrieveRsp->numOfRows); + code = + extractDataBlockFromFetchRsp(pb, pLoadInfo, pRetrieveRsp->numOfRows, pStart, + pRetrieveRsp->compLen, pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); + if (code != 0) { + taosMemoryFreeClear(pDataInfo->pRsp); + goto _error; + } + + taosArrayPush(pExchangeInfo->pResultBlockList, &pb); } + pLoadInfo->totalElapsed += (taosGetTimestampUs() - startTs); + if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d" - " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 - ", completed:%d try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources); + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d" + " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb," + " completed:%d try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfRows, + pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0, completed + 1, i + 1, totalSources); completed += 1; pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 - ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, - pLoadInfo->totalRows, pLoadInfo->totalSize); + ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, + pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0); } taosMemoryFreeClear(pDataInfo->pRsp); @@ -2191,11 +2194,12 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx } } - return pExchangeInfo->pResult; + return; } if (completed == totalSources) { - return setAllSourcesCompleted(pOperator, startTs); + setAllSourcesCompleted(pOperator, startTs); + return; } sched_yield(); @@ -2203,7 +2207,6 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx _error: pTaskInfo->code = code; - return NULL; } static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { @@ -2233,7 +2236,7 @@ static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { +static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2242,7 +2245,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { while (1) { if (pExchangeInfo->current >= totalSources) { - return setAllSourcesCompleted(pOperator, startTs); + setAllSourcesCompleted(pOperator, startTs); + return TSDB_CODE_SUCCESS; } doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); @@ -2255,7 +2259,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); pOperator->pTaskInfo->code = pDataInfo->code; - return NULL; + return pOperator->pTaskInfo->code; } SRetrieveTableRsp* pRsp = pDataInfo->pRsp; @@ -2272,16 +2276,16 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { continue; } - SSDataBlock* pRes = pExchangeInfo->pResult; - SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; - int32_t code = - extractDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + + char* pStart = pRetrieveRsp->data; + int32_t code = extractDataBlockFromFetchRsp(NULL, pLoadInfo, pRetrieveRsp->numOfRows, pStart, pRetrieveRsp->compLen, + pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources); @@ -2290,13 +2294,13 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize); } - pOperator->resultInfo.totalRows += pRes->info.rows; + pOperator->resultInfo.totalRows += pRetrieveRsp->numOfRows; taosMemoryFreeClear(pDataInfo->pRsp); - return pExchangeInfo->pResult; + return TSDB_CODE_SUCCESS; } } @@ -2320,6 +2324,11 @@ static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } +static void freeBlock(void* pParam) { + SSDataBlock* pBlock = *(SSDataBlock**)pParam; + blockDataDestroy(pBlock); +} + static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2329,9 +2338,9 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { return NULL; } - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pOperator->status == OP_EXEC_DONE) { qDebug("%s all %" PRIzu " source(s) are exhausted, total rows:%" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, @@ -2339,11 +2348,23 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { return NULL; } - if (pExchangeInfo->seqLoadData) { - return seqLoadRemoteData(pOperator); - } else { - return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList); + if (size == 0 || pExchangeInfo->rspBlockIndex >= size) { + pExchangeInfo->rspBlockIndex = 0; + taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock); + if (pExchangeInfo->seqLoadData) { + seqLoadRemoteData(pOperator); + } else { + concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + } + + if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { + return NULL; + } } + + // we have buffered retrieved datablock, return it directly + return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++); } static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { @@ -2360,26 +2381,24 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { return NULL; } - ASSERT(pBlock == pExchangeInfo->pResult); - SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; if (hasLimitOffsetInfo(pLimitInfo)) { - int32_t status = handleLimitOffset(pOperator, pLimitInfo, pExchangeInfo->pResult, false); + int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); if (status == PROJECT_RETRIEVE_CONTINUE) { continue; } else if (status == PROJECT_RETRIEVE_DONE) { - size_t rows = pExchangeInfo->pResult->info.rows; + size_t rows = pBlock->info.rows; pExchangeInfo->limitInfo.numOfOutputRows += rows; if (rows == 0) { doSetOperatorCompleted(pOperator); return NULL; } else { - return pExchangeInfo->pResult; + return pBlock; } } } else { - return pExchangeInfo->pResult; + return pBlock; } } } @@ -2442,16 +2461,18 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode } tsem_init(&pInfo->ready, 0, 0); + pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES); pInfo->seqLoadData = false; pInfo->pTransporter = pTransporter; - pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pResult->pDataBlock); + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, @@ -3585,12 +3606,15 @@ void doDestroyExchangeOperatorInfo(void* param) { taosArrayDestroy(pExInfo->pSources); taosArrayDestroy(pExInfo->pSourceDataInfo); - if (pExInfo->pResult != NULL) { - pExInfo->pResult = blockDataDestroy(pExInfo->pResult); + + if (pExInfo->pResultBlockList != NULL) { + taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock); + pExInfo->pResultBlockList = NULL; } - tsem_destroy(&pExInfo->ready); + blockDataDestroy(pExInfo->pDummyBlock); + tsem_destroy(&pExInfo->ready); taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8545abecfb..03c08a6bf5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2266,9 +2266,11 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } } + char* pStart = pRsp->data; extractDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pRsp->numOfRows, pRsp->data, pRsp->compLen, - pOperator->exprSupp.numOfExprs, startTs, NULL, pInfo->scanCols); + pOperator->exprSupp.numOfExprs, NULL, pInfo->scanCols, &pStart); + //startTs, // todo log the filter info doFilterResult(pInfo); taosMemoryFree(pRsp); diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 0bf6d4c921..c96854b198 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -27,10 +27,9 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock ASSERT(pReq->blockNum == taosArrayGetSize(pReq->dataLen)); for (int32_t i = 0; i < blockNum; i++) { - /*int32_t len = *(int32_t*)taosArrayGet(pReq->dataLen, i);*/ SRetrieveTableRsp* pRetrieve = taosArrayGetP(pReq->data, i); SSDataBlock* pDataBlock = taosArrayGet(pArray, i); - blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); + blockDecode(pDataBlock, pRetrieve->data); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); @@ -51,7 +50,7 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock taosArraySetSize(pArray, 1); SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); - blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); + blockDecode(pDataBlock, pRetrieve->data); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); From d0524389d9501723f48514255636358e437a90ce Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 5 Aug 2022 15:07:38 +0800 Subject: [PATCH 31/96] fix: plan problem with interval fill statement --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 1ebba8bb63..2e2c4fbff7 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -719,7 +719,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; - pWindow->node.groupAction = getGroupAction(pCxt, pSelect); + pWindow->node.groupAction = (NULL != pInterval->pFill ? GROUP_ACTION_KEEP : getGroupAction(pCxt, pSelect)); pWindow->node.requireDataOrder = pSelect->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_IN_BLOCK; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; From 0ac1b85a0e72b399a08f56bbefd25c9634c8a4e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 15:09:43 +0800 Subject: [PATCH 32/96] refactor: add some logs. --- source/libs/executor/src/executorimpl.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8dfda896cc..d17fe7633f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1959,7 +1959,8 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfRows); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows); } else { pSourceDataInfo->code = code; qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); @@ -2171,16 +2172,17 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d" - " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb," + " index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb," " completed:%d try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfRows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0, completed + 1, i + 1, totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, + pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, + completed + 1, i + 1, totalSources); completed += 1; pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, - pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0); + pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0); } taosMemoryFreeClear(pDataInfo->pRsp); From 52173e14e1b753f4a530f205f05024b01e89ff55 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 16:10:45 +0800 Subject: [PATCH 33/96] fix(query): add check for null ptr. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2a8e234ac0..0749d7b133 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2312,6 +2312,12 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe return; } else { // has next point in mem/imem TSDBROW* pNextRow = getValidRow(pIter, pDelList, pReader); + if (pNextRow == NULL) { + *pTSRow = current.pTSRow; + *freeTSRow = false; + return; + } + if (TSDBROW_KEY(¤t).ts != TSDBROW_KEY(pNextRow).ts) { *pTSRow = current.pTSRow; *freeTSRow = false; From 1063064a59916219b24605d0b89ddc02552ad747 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 17:40:40 +0800 Subject: [PATCH 34/96] enh(query): support filter by SMA. --- include/libs/scalar/filter.h | 2 +- source/libs/executor/src/executorimpl.c | 12 +-- source/libs/executor/src/scanoperator.c | 116 ++++++++++++++++-------- source/libs/scalar/src/filter.c | 12 +-- 4 files changed, 84 insertions(+), 58 deletions(-) diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index 44064f0a9f..1f1d9dea93 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -44,7 +44,7 @@ extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo); extern void filterFreeInfo(SFilterInfo *info); -extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); +extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t numOfCols, int32_t numOfRows); /* condition split interface */ int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d17fe7633f..1908e2a92d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1011,16 +1011,6 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { // } //} -// static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, -// SqlFunctionCtx *pCtx, int32_t numOfRows) { -// STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -// -// if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { -// return true; -// } -// -// return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows); -// } #if 0 static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { STimeWindow w = {0}; @@ -1215,7 +1205,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc } // current block has been discard due to filter applied -// if (!doFilterByBlockStatistics(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) { +// if (!doFilterByBlockSMA(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) { // pCost->skipBlocks += 1; // qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, // pBlockInfo->window.ekey, pBlockInfo->rows); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 03c08a6bf5..a2e0b68623 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,10 +13,11 @@ * along with this program. If not, see . */ +#include "os.h" #include "executorimpl.h" +#include "filter.h" #include "function.h" #include "functionMgt.h" -#include "os.h" #include "querynodes.h" #include "systable.h" #include "tname.h" @@ -227,6 +228,57 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* return TSDB_CODE_SUCCESS; } +static FORCE_INLINE bool doFilterByBlockSMA(const SNode* pFilterNode, SColumnDataAgg** pColsAgg, int32_t numOfCols, + int32_t numOfRows) { + if (pColsAgg == NULL || pFilterNode == NULL) { + return true; + } + + SFilterInfo* filter = NULL; + + // todo move to the initialization function + int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + bool keep = filterRangeExecute(filter, pColsAgg, numOfCols, numOfRows); + + filterFreeInfo(filter); + return keep; +} + +static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + bool allColumnsHaveAgg = true; + SColumnDataAgg** pColAgg = NULL; + + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + if (!allColumnsHaveAgg) { + return false; + } + + // if (allColumnsHaveAgg == true) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + + // todo create this buffer during creating operator + if (pBlock->pBlockAgg == NULL) { + pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); + if (pBlock->pBlockAgg == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + } + } + + for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->pColMatchInfo); ++i) { + SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); + if (!pColMatchInfo->output) { + continue; + } + pBlock->pBlockAgg[pColMatchInfo->targetSlotId] = pColAgg[i]; + } + + return true; +} + static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -236,6 +288,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pCost->totalBlocks += 1; pCost->totalRows += pBlock->info.rows; + bool loadSMA = false; *status = pInfo->dataBlockLoadFlag; if (pTableScanInfo->pFilterNode != NULL || @@ -259,41 +312,34 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; - - bool allColumnsHaveAgg = true; - SColumnDataAgg** pColAgg = NULL; - - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } - - if (allColumnsHaveAgg == true) { - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - - // todo create this buffer during creating operator - if (pBlock->pBlockAgg == NULL) { - pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); - } - - for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->pColMatchInfo); ++i) { - SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); - if (!pColMatchInfo->output) { - continue; - } - pBlock->pBlockAgg[pColMatchInfo->targetSlotId] = pColAgg[i]; - } - - return TSDB_CODE_SUCCESS; - } else { // failed to load the block sma data, data block statistics does not exist, load data block instead + loadSMA = true; // mark the operator of load sma; + bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); + if (!success) { // failed to load the block sma data, data block statistics does not exist, load data block instead *status = FUNC_DATA_REQUIRED_DATA_LOAD; } } ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD); - // todo filter data block according to the block sma data firstly + // try to filter data block according to sma info + if (pTableScanInfo->pFilterNode != NULL) { + if (!loadSMA) { + doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); + } + bool keep = doFilterByBlockSMA(pTableScanInfo->pFilterNode, pBlock->pBlockAgg, taosArrayGetSize(pBlock->pDataBlock), + pBlockInfo->rows); + if (!keep) { + qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + pCost->filterOutBlocks += 1; + (*status) = FUNC_DATA_REQUIRED_FILTEROUT; + + return TSDB_CODE_SUCCESS; + } + } + + // try to filter datablock according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), @@ -303,16 +349,6 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return TSDB_CODE_SUCCESS; } -#if 0 - if (!doFilterByBlockStatistics(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { - pCost->filterOutBlocks += 1; - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); - (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - return TSDB_CODE_SUCCESS; - } -#endif - pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; @@ -2722,7 +2758,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc // todo filter data block according to the block sma data firstly #if 0 - if (!doFilterByBlockStatistics(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { + if (!doFilterByBlockSMA(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { pCost->filterOutBlocks += 1; qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 04328fda9c..a6a47da2e5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3245,7 +3245,7 @@ _return: return code; } -bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows) { +bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { if (FILTER_EMPTY_RES(info)) { return false; } @@ -3261,7 +3261,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t int32_t index = -1; SFilterRangeCtx *ctx = info->colRange[k]; for(int32_t i = 0; i < numOfCols; ++i) { - if (pDataStatis[i].colId == ctx->colId) { + if (pDataStatis[i]->colId == ctx->colId) { index = i; break; } @@ -3277,13 +3277,13 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t break; } - if (pDataStatis[index].numOfNull <= 0) { + if (pDataStatis[index]->numOfNull <= 0) { if (ctx->isnull && !ctx->notnull && !ctx->isrange) { ret = false; break; } - } else if (pDataStatis[index].numOfNull > 0) { - if (pDataStatis[index].numOfNull == numOfRows) { + } else if (pDataStatis[index]->numOfNull > 0) { + if (pDataStatis[index]->numOfNull == numOfRows) { if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { ret = false; break; @@ -3297,7 +3297,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t } } - SColumnDataAgg* pDataBlockst = &pDataStatis[index]; + SColumnDataAgg* pDataBlockst = pDataStatis[index]; SFilterRangeNode *r = ctx->rs; float minv = 0; From c7b511ed698b8addf86dc822d29ce961ba8e020e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 6 Aug 2022 16:13:18 +0800 Subject: [PATCH 35/96] fix(query): set src block for function. --- source/libs/executor/src/executorimpl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1908e2a92d..14914b51c0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -434,6 +434,7 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC pCtx[i].order = order; pCtx[i].input.numOfRows = pBlock->info.rows; setBlockStatisInfo(&pCtx[i], &pOperator->exprSupp.pExprInfo[i], pBlock); + pCtx[i].pSrcBlock = pBlock; } } From ebe8192d7ecd526546c0eab9a5c11087de5bda88 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 6 Aug 2022 16:15:15 +0800 Subject: [PATCH 36/96] refactor(query): do some internal refactor. --- source/libs/executor/src/executorimpl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 14914b51c0..b0929a9acd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -88,7 +88,7 @@ static int32_t getExprFunctionId(SExprInfo* pExprInfo) { static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); -static void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pSDataBlock); +static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pSDataBlock); static void releaseQueryBuf(size_t numOfTables); @@ -433,7 +433,7 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { pCtx[i].order = order; pCtx[i].input.numOfRows = pBlock->info.rows; - setBlockStatisInfo(&pCtx[i], &pOperator->exprSupp.pExprInfo[i], pBlock); + setBlockSMAInfo(&pCtx[i], &pOperator->exprSupp.pExprInfo[i], pBlock); pCtx[i].pSrcBlock = pBlock; } } @@ -799,7 +799,7 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF return TSDB_CODE_SUCCESS; } -void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pBlock) { +void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pBlock) { int32_t numOfRows = pBlock->info.rows; SInputColumnInfoData* pInput = &pCtx->input; From 5ae9fdff7a59bee3c685c147d142235650335770 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 8 Aug 2022 10:00:10 +0800 Subject: [PATCH 37/96] enh: increase explain output, enhance debugging function --- source/libs/command/inc/commandInt.h | 5 +++ source/libs/command/src/explain.c | 31 +++++++++++++++-- source/libs/function/src/builtins.c | 7 ++-- source/libs/parser/src/parInsertData.c | 6 ++-- source/libs/parser/src/parTranslater.c | 2 -- source/libs/planner/src/planLogicCreater.c | 40 ++++++++++++++-------- source/libs/planner/src/planPhysiCreater.c | 2 +- 7 files changed, 67 insertions(+), 26 deletions(-) diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index c53d464eab..e2288ed3ef 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -43,6 +43,7 @@ extern "C" { #define EXPLAIN_GROUP_SORT_FORMAT "Group Sort" #define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s" #define EXPLAIN_MERGE_INTERVAL_FORMAT "Merge Interval on Column %s" +#define EXPLAIN_MERGE_ALIGNED_INTERVAL_FORMAT "Merge Aligned Interval on Column %s" #define EXPLAIN_FILL_FORMAT "Fill" #define EXPLAIN_SESSION_FORMAT "Session" #define EXPLAIN_STATE_WINDOW_FORMAT "StateWindow on Column %s" @@ -59,6 +60,8 @@ extern "C" { #define EXPLAIN_RATIO_TIME_FORMAT "Ratio: %f" #define EXPLAIN_MERGE_FORMAT "SortMerge" #define EXPLAIN_MERGE_KEYS_FORMAT "Merge Key: " +#define EXPLAIN_PARTITION_KETS_FORMAT "Partition Key: " +#define EXPLAIN_INTERP_FORMAT "Interp" #define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms" #define EXPLAIN_EXEC_TIME_FORMAT "Execution Time: %.3f ms" @@ -69,6 +72,7 @@ extern "C" { #define EXPLAIN_LEFT_PARENTHESIS_FORMAT " (" #define EXPLAIN_RIGHT_PARENTHESIS_FORMAT ")" #define EXPLAIN_BLANK_FORMAT " " +#define EXPLAIN_COMMA_FORMAT ", " #define EXPLAIN_COST_FORMAT "cost=%.2f..%.2f" #define EXPLAIN_ROWS_FORMAT "rows=%" PRIu64 #define EXPLAIN_COLUMNS_FORMAT "columns=%d" @@ -86,6 +90,7 @@ extern "C" { #define EXPLAIN_OUTPUT_ORDER_TYPE_FORMAT "output_order=%s" #define EXPLAIN_OFFSET_FORMAT "offset=%d" #define EXPLAIN_SOFFSET_FORMAT "soffset=%d" +#define EXPLAIN_PARTITIONS_FORMAT "partitions=%d" #define COMMAND_RESET_LOG "resetLog" #define COMMAND_SCHEDULE_POLICY "schedulePolicy" diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 2836bd7acd..5935b19173 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -535,6 +535,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (NULL != pTblScanNode->pGroupTags) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PARTITION_KETS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_PARTITIONS_FORMAT, pTblScanNode->pGroupTags->length); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, @@ -612,13 +619,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pPrjNode->mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pPrjNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -915,7 +926,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: { SMergeAlignedIntervalPhysiNode *pIntNode = (SMergeAlignedIntervalPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); + EXPLAIN_ROW_NEW(level, EXPLAIN_MERGE_ALIGNED_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); @@ -1129,6 +1140,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PARTITION_KETS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_PARTITIONS_FORMAT, pPartNode->pPartitionKeys->length); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pPartNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pPartNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, @@ -1201,11 +1217,20 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGE_KEYS_FORMAT); + if (pMergeNode->groupSort) { + EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, "_group_id asc"); + if (LIST_LENGTH(pMergeNode->pMergeKeys) > 0) { + EXPLAIN_ROW_APPEND(EXPLAIN_COMMA_FORMAT); + } + } for (int32_t i = 0; i < LIST_LENGTH(pMergeNode->pMergeKeys); ++i) { SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pMergeNode->pMergeKeys, i); EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, nodesGetNameFromColumnNode(ptn->pExpr)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, EXPLAIN_ORDER_STRING(ptn->order)); + if (i != LIST_LENGTH(pMergeNode->pMergeKeys) - 1) { + EXPLAIN_ROW_APPEND(EXPLAIN_COMMA_FORMAT); + } } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -1417,7 +1442,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: { SInterpFuncPhysiNode *pInterpNode = (SInterpFuncPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT); + EXPLAIN_ROW_NEW(level, EXPLAIN_INTERP_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bdd9232ce3..13ebcd4518 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1575,7 +1575,8 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType && TSDB_DATA_TYPE_TIMESTAMP != colType) { + if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType && + TSDB_DATA_TYPE_TIMESTAMP != colType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -2150,7 +2151,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "_apercentile_partial", - .type = FUNCTION_TYPE_APERCENTILE_PARTIAL | FUNC_MGT_TIMELINE_FUNC, + .type = FUNCTION_TYPE_APERCENTILE_PARTIAL, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateApercentilePartial, .getEnvFunc = getApercentileFuncEnv, @@ -2163,7 +2164,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_apercentile_merge", .type = FUNCTION_TYPE_APERCENTILE_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateApercentileMerge, .getEnvFunc = getApercentileFuncEnv, .initFunc = apercentileFunctionSetup, diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index ee35b23cc0..cd30672baf 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -486,11 +486,11 @@ static int32_t tdBlockRowMerge(STableMeta* pTableMeta, SBlockKeyTuple* pEndKeyTp void* pDestRow = NULL; ++((*pBlkRowMerger)->index); if ((*pBlkRowMerger)->index < taosArrayGetSize((*pBlkRowMerger)->rowArray)) { - void* pAlloc = *(void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index); - if (tRealloc((uint8_t**)&pAlloc, rowSize) != 0) { + void** pAlloc = (void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index); + if (tRealloc((uint8_t**)pAlloc, rowSize) != 0) { return TSDB_CODE_FAILED; } - pDestRow = pAlloc; + pDestRow = *pAlloc; } else { if (tRealloc((uint8_t**)&pDestRow, rowSize) != 0) { return TSDB_CODE_FAILED; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d96cdca8a9..2f6725acaf 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2543,7 +2543,6 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } - pSelect->isTimeLineResult = true; pCxt->currClause = SQL_CLAUSE_WINDOW; int32_t code = translateExpr(pCxt, &pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { @@ -2614,7 +2613,6 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec if (NULL == pSelect->pPartitionByList) { return TSDB_CODE_SUCCESS; } - pSelect->isTimeLineResult = false; pCxt->currClause = SQL_CLAUSE_PARTITION_BY; return translateExprList(pCxt, pSelect->pPartitionByList); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 2e2c4fbff7..98b97765de 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -342,7 +342,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pJoin->inputTsOrder = ORDER_ASC; pJoin->node.groupAction = GROUP_ACTION_CLEAR; pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; - pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; + pJoin->node.resultDataOrder = DATA_ORDER_LEVEL_GLOBAL; int32_t code = TSDB_CODE_SUCCESS; @@ -472,6 +472,11 @@ static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; } +static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { + return needTimeline ? (NULL != pSelect->pPartitionByList ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) + : DATA_ORDER_LEVEL_NONE; +} + static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; @@ -486,8 +491,8 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->hasTimeLineFunc = pSelect->hasTimeLineFunc; pAgg->onlyHasKeepOrderFunc = pSelect->onlyHasKeepOrderFunc; pAgg->node.groupAction = getGroupAction(pCxt, pSelect); - pAgg->node.requireDataOrder = pAgg->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; - pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; + pAgg->node.requireDataOrder = getRequireDataOrder(pAgg->hasTimeLineFunc, pSelect); + pAgg->node.resultDataOrder = pAgg->onlyHasKeepOrderFunc ? pAgg->node.requireDataOrder : DATA_ORDER_LEVEL_NONE; int32_t code = TSDB_CODE_SUCCESS; @@ -557,8 +562,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt pIdfRowsFunc->isUniqueFunc = pSelect->hasUniqueFunc; pIdfRowsFunc->isTimeLineFunc = pSelect->hasTimeLineFunc; pIdfRowsFunc->node.groupAction = getGroupAction(pCxt, pSelect); - pIdfRowsFunc->node.requireDataOrder = - pIdfRowsFunc->isTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; + pIdfRowsFunc->node.requireDataOrder = getRequireDataOrder(pIdfRowsFunc->isTimeLineFunc, pSelect); pIdfRowsFunc->node.resultDataOrder = pIdfRowsFunc->node.requireDataOrder; // indefinite rows functions and _select_values functions @@ -592,7 +596,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p } pInterpFunc->node.groupAction = getGroupAction(pCxt, pSelect); - pInterpFunc->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pInterpFunc->node.requireDataOrder = getRequireDataOrder(true, pSelect); pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder; int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsInterpFunc, &pInterpFunc->pFuncs); @@ -668,8 +672,10 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo pWindow->winType = WINDOW_TYPE_STATE; pWindow->node.groupAction = getGroupAction(pCxt, pSelect); - pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; - pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : getRequireDataOrder(true, pSelect); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : pWindow->node.requireDataOrder; pWindow->pStateExpr = nodesCloneNode(pState->pExpr); pWindow->pTspk = nodesCloneNode(pState->pCol); if (NULL == pWindow->pStateExpr || NULL == pWindow->pTspk) { @@ -691,8 +697,10 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i; pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? SESSION_ALGO_STREAM_SINGLE : SESSION_ALGO_MERGE; pWindow->node.groupAction = getGroupAction(pCxt, pSelect); - pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; - pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : getRequireDataOrder(true, pSelect); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : pWindow->node.requireDataOrder; pWindow->pTspk = nodesCloneNode((SNode*)pSession->pCol); if (NULL == pWindow->pTspk) { @@ -720,8 +728,12 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; pWindow->node.groupAction = (NULL != pInterval->pFill ? GROUP_ACTION_KEEP : getGroupAction(pCxt, pSelect)); - pWindow->node.requireDataOrder = pSelect->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_IN_BLOCK; - pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery + ? DATA_ORDER_LEVEL_IN_BLOCK + : (pSelect->hasTimeLineFunc ? getRequireDataOrder(true, pSelect) : DATA_ORDER_LEVEL_IN_BLOCK); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : getRequireDataOrder(true, pSelect); pWindow->pTspk = nodesCloneNode(pInterval->pCol); if (NULL == pWindow->pTspk) { @@ -768,8 +780,8 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } pFill->node.groupAction = getGroupAction(pCxt, pSelect); - pFill->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; - pFill->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pFill->node.requireDataOrder = getRequireDataOrder(true, pSelect); + pFill->node.resultDataOrder = pFill->node.requireDataOrder; pFill->inputTsOrder = ORDER_ASC; int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_WINDOW, NULL, COLLECT_COL_TYPE_ALL, &pFill->node.pTargets); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 4fe229269a..ff2e2f1494 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -985,7 +985,7 @@ static bool projectCanMergeDataBlock(SProjectLogicNode* pProject) { return true; } if (1 != LIST_LENGTH(pProject->node.pChildren)) { - return false; + return true; } SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProject->node.pChildren, 0); return DATA_ORDER_LEVEL_GLOBAL == pChild->resultDataOrder ? true : false; From 447b2cb9f1b8cd84e6e1c75890488c0ee9367e3d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Aug 2022 12:56:03 +0800 Subject: [PATCH 38/96] fix(query): update the load data log. --- source/libs/executor/inc/executorimpl.h | 6 ++-- source/libs/executor/src/executorimpl.c | 37 ++++++++++++------------- source/libs/executor/src/scanoperator.c | 5 ++-- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 62b2cc0fa6..1b8be3b53c 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -858,8 +858,10 @@ void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLi void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart); +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, char** pNextStart); +void updateLoadRemoteInfo(SLoadRemoteDataInfo *pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator); + STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b0929a9acd..752b93bcea 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2035,8 +2035,16 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf return TSDB_CODE_SUCCESS; } -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart) { +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator) { + pInfo->totalRows += numOfRows; + pInfo->totalSize += dataLen; + pInfo->totalElapsed += (taosGetTimestampUs() - startTs); + pOperator->resultInfo.totalRows += numOfRows; +} + +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, + char** pNextStart) { if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); *pNextStart = (char*) blockDecode(pRes, pData); @@ -2064,24 +2072,16 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo } blockDecode(pBlock, pStart); - blockDataEnsureCapacity(pRes, numOfRows); + blockDataEnsureCapacity(pRes, pBlock->info.rows); // data from mnode - pRes->info.rows = numOfRows; + pRes->info.rows = pBlock->info.rows; relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); blockDataDestroy(pBlock); } // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. blockDataUpdateTsWindow(pRes, 0); - - pLoadInfo->totalRows += numOfRows; - pLoadInfo->totalSize += compLen; - - if (total != NULL) { - *total += numOfRows; - } - return TSDB_CODE_SUCCESS; } @@ -2148,9 +2148,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn while(index++ < pRetrieveRsp->numOfBlocks) { SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); blockDataEnsureCapacity(pb, pRetrieveRsp->numOfRows); - code = - extractDataBlockFromFetchRsp(pb, pLoadInfo, pRetrieveRsp->numOfRows, pStart, - pRetrieveRsp->compLen, pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); + code = extractDataBlockFromFetchRsp(pb, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); goto _error; @@ -2159,7 +2157,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn taosArrayPush(pExchangeInfo->pResultBlockList, &pb); } - pLoadInfo->totalElapsed += (taosGetTimestampUs() - startTs); + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d" @@ -2272,8 +2270,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; char* pStart = pRetrieveRsp->data; - int32_t code = extractDataBlockFromFetchRsp(NULL, pLoadInfo, pRetrieveRsp->numOfRows, pStart, pRetrieveRsp->compLen, - pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); + int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 @@ -2291,7 +2288,9 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { pLoadInfo->totalRows, pLoadInfo->totalSize); } - pOperator->resultInfo.totalRows += pRetrieveRsp->numOfRows; + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); + pDataInfo->totalRows += pRetrieveRsp->numOfRows; + taosMemoryFreeClear(pDataInfo->pRsp); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a2e0b68623..0bd0d68cae 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2303,10 +2303,9 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } char* pStart = pRsp->data; - extractDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pRsp->numOfRows, pRsp->data, pRsp->compLen, - pOperator->exprSupp.numOfExprs, NULL, pInfo->scanCols, &pStart); + extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pOperator->exprSupp.numOfExprs, pInfo->scanCols, &pStart); + updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); - //startTs, // todo log the filter info doFilterResult(pInfo); taosMemoryFree(pRsp); From 200a669542175755a1f6d4bc669f41f2d4760047 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Aug 2022 16:34:11 +0800 Subject: [PATCH 39/96] refactor: do some internal refactor. --- include/common/tmsg.h | 6 ++--- source/client/src/clientImpl.c | 6 ++++- source/common/src/tdatablock.c | 34 +++++++++++++++++-------- source/libs/executor/src/executorimpl.c | 6 ++++- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9d001c9534..5ca6a337a8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -296,13 +296,13 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_IDX_ON ((int8_t)0x2) #define COL_SET_NULL ((int8_t)0x10) #define COL_SET_VAL ((int8_t)0x20) -typedef struct SSchema { +struct SSchema { int8_t type; int8_t flags; col_id_t colId; int32_t bytes; char name[TSDB_COL_NAME_LEN]; -} SSchema; +}; #define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0) #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) @@ -648,7 +648,7 @@ typedef struct { }; bool output; // TODO remove it later - int16_t type; + int8_t type; int32_t bytes; uint8_t precision; uint8_t scale; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8357e2d627..480fd99c81 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1762,6 +1762,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 char* p = (char*)pResultInfo->pData; + // version: + int32_t blockVersion = *(int32_t*)p; + p += sizeof(int32_t); + int32_t dataLen = *(int32_t*)p; p += sizeof(int32_t); @@ -1782,7 +1786,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 // check fields for (int32_t i = 0; i < numOfCols; ++i) { int16_t type = *(int16_t*)p; - p += sizeof(int16_t); + p += sizeof(int8_t); int32_t bytes = *(int32_t*)p; p += sizeof(int32_t); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index bcbdf8d786..45bbe96923 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -676,9 +676,9 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { * @return */ size_t blockDataGetSerialMetaSize(uint32_t numOfCols) { - // | total length | total rows | total columns | has column seg| block group id | column schema | each column length | - return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + - numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | + return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + + numOfCols * (sizeof(int8_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); } double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { @@ -1582,7 +1582,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { for (int32_t i = 0; i < sz; i++) { SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); tlen += taosEncodeFixedI16(buf, pColData->info.colId); - tlen += taosEncodeFixedI16(buf, pColData->info.type); + tlen += taosEncodeFixedI8(buf, pColData->info.type); tlen += taosEncodeFixedI32(buf, pColData->info.bytes); tlen += taosEncodeFixedBool(buf, pColData->hasNull); @@ -1614,7 +1614,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { for (int32_t i = 0; i < sz; i++) { SColumnInfoData data = {0}; buf = taosDecodeFixedI16(buf, &data.info.colId); - buf = taosDecodeFixedI16(buf, &data.info.type); + buf = taosDecodeFixedI8(buf, &data.info.type); buf = taosDecodeFixedI32(buf, &data.info.bytes); buf = taosDecodeFixedBool(buf, &data.hasNull); @@ -2074,6 +2074,10 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { // todo extract method + int32_t* version = (int32_t*)data; + *version = 1; + data += sizeof(int32_t); + int32_t* actualLen = (int32_t*)data; data += sizeof(int32_t); @@ -2085,8 +2089,11 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ *cols = numOfCols; data += sizeof(int32_t); - int32_t* hasColumnSegment = (int32_t*)data; - *hasColumnSegment = 1; + // flag segment. + // the inital bit is for column info + int32_t* flagSegment = (int32_t*)data; + *flagSegment = (1<<31); + data += sizeof(int32_t); uint64_t* groupId = (uint64_t*)data; @@ -2095,8 +2102,8 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - *((int16_t*)data) = pColInfoData->info.type; - data += sizeof(int16_t); + *((int8_t*)data) = pColInfoData->info.type; + data += sizeof(int8_t); *((int32_t*)data) = pColInfoData->info.bytes; data += sizeof(int32_t); @@ -2145,6 +2152,10 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { const char* pStart = pData; + int32_t version = *(int32_t*) pStart; + pStart += sizeof(int32_t); + ASSERT(version == 1); + // total length sizeof(int32_t) int32_t dataLen = *(int32_t*)pStart; pStart += sizeof(int32_t); @@ -2158,7 +2169,8 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(int32_t); // has column info segment - int32_t hasColumnInfo = *(int32_t*)pStart; + int32_t flagSeg = *(int32_t*)pStart; + int32_t hasColumnInfo = (flagSeg >> 31); pStart += sizeof(int32_t); // group id sizeof(uint64_t) @@ -2173,7 +2185,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData->info.type = *(int16_t*)pStart; - pStart += sizeof(int16_t); + pStart += sizeof(int8_t); pColInfoData->info.bytes = *(int32_t*)pStart; pStart += sizeof(int32_t); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 752b93bcea..b7e099d0b1 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4071,7 +4071,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else { ASSERT(0); } - pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; + + if (pOperator != NULL) { + pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; + } + return pOperator; } From 0aef734a20c412f3c6ff00a526dffc3ccebb020b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Aug 2022 17:56:26 +0800 Subject: [PATCH 40/96] fix(query): fix syntax error. --- source/libs/executor/src/executorimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 844ee4cc6e..5f5c035523 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1517,7 +1517,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); - doUpdateNumOfRows(pRow, numOfExprs, rowEntryOffset); + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); // no results, continue to check the next one if (pRow->numOfRows == 0) { From 7e1917f105539a5e89563c01a72dea893e5d5d1b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 09:55:03 +0800 Subject: [PATCH 41/96] enh: refine the data requirements of the partition physical plan --- source/libs/function/src/builtins.c | 12 ++++++------ source/libs/planner/src/planOptimizer.c | 4 ++-- source/libs/planner/src/planUtil.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 13ebcd4518..9102137c31 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2452,7 +2452,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "histogram", .type = FUNCTION_TYPE_HISTOGRAM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateHistogram, .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, @@ -2467,7 +2467,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_histogram_partial", .type = FUNCTION_TYPE_HISTOGRAM_PARTIAL, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateHistogramPartial, .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, @@ -2479,7 +2479,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_histogram_merge", .type = FUNCTION_TYPE_HISTOGRAM_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateHistogramMerge, .getEnvFunc = getHistogramFuncEnv, .initFunc = functionSetup, @@ -2491,7 +2491,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "hyperloglog", .type = FUNCTION_TYPE_HYPERLOGLOG, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateHLL, .getEnvFunc = getHLLFuncEnv, .initFunc = functionSetup, @@ -2505,7 +2505,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "_hyperloglog_partial", - .type = FUNCTION_TYPE_HYPERLOGLOG_PARTIAL | FUNC_MGT_TIMELINE_FUNC, + .type = FUNCTION_TYPE_HYPERLOGLOG_PARTIAL, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateHLLPartial, .getEnvFunc = getHLLFuncEnv, @@ -2517,7 +2517,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "_hyperloglog_merge", - .type = FUNCTION_TYPE_HYPERLOGLOG_MERGE | FUNC_MGT_TIMELINE_FUNC, + .type = FUNCTION_TYPE_HYPERLOGLOG_MERGE, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateHLLMerge, .getEnvFunc = getHLLFuncEnv, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 86a4441cff..0e83051538 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1711,7 +1711,7 @@ static bool eliminateProjOptCanChildConditionUseChildTargets(SLogicNode* pChild, if (!cxt.canUse) return false; } if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pChild) && NULL != ((SJoinLogicNode*)pChild)->pOnConditions) { - SJoinLogicNode* pJoinLogicNode = (SJoinLogicNode*)pChild; + SJoinLogicNode* pJoinLogicNode = (SJoinLogicNode*)pChild; CheckNewChildTargetsCxt cxt = {.pNewChildTargets = pNewChildTargets, .canUse = false}; nodesWalkExpr(pJoinLogicNode->pOnConditions, eliminateProjOptCanUseNewChildTargetsImpl, &cxt); if (!cxt.canUse) return false; @@ -1768,7 +1768,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pProjectNode->node.pChildren); nodesDestroyNode((SNode*)pProjectNode); - //if pChild is a project logic node, remove its projection which is not reference by its target. + // if pChild is a project logic node, remove its projection which is not reference by its target. alignProjectionWithTarget(pChild); } pCxt->optimized = true; diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index d7310892de..ac1589bec0 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -232,7 +232,8 @@ static int32_t adjustPartitionDataRequirement(SPartitionLogicNode* pPart, EDataO return TSDB_CODE_PLAN_INTERNAL_ERROR; } pPart->node.resultDataOrder = requirement; - pPart->node.requireDataOrder = requirement; + pPart->node.requireDataOrder = + (requirement >= DATA_ORDER_LEVEL_IN_BLOCK ? DATA_ORDER_LEVEL_GLOBAL : DATA_ORDER_LEVEL_NONE); return TSDB_CODE_SUCCESS; } From 30a64bdcd9315dedbd69c0680ec38e9a5b508273 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 10:16:00 +0800 Subject: [PATCH 42/96] fix(query): set correct schema. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 139 ++++++++++++------------- 1 file changed, 68 insertions(+), 71 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0749d7b133..e1e236bf11 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -14,6 +14,7 @@ */ #include "tsdb.h" +#include "osDef.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) typedef enum { @@ -129,7 +130,8 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; - STSchema* pSchema; + STSchema* pSchema;// the newest version schema + STSchema* pMemSchema;// the previous schema for in-memory data, to avoid load schema too many times SDataFReader* pFileReader; SVersionRange verRange; @@ -145,11 +147,10 @@ static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanI SRowMerger* pMerger); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); -static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow); +static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); static int32_t doAppendRowFromBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); -static void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); static void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, @@ -1230,6 +1231,31 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB return false; } +static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { + // always set the newest schema version in pReader->pSchema + if (pReader->pSchema == NULL) { + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1); + } + + if (sversion == pReader->pSchema->version) { + return pReader->pSchema; + } + + if (pReader->pMemSchema == NULL) { + int32_t code = + metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); + return pReader->pMemSchema; + } + + if (pReader->pMemSchema->version == sversion) { + return pReader->pMemSchema; + } + + taosMemoryFree(pReader->pMemSchema); + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); + return pReader->pMemSchema; +} + static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key) { SRowMerger merge = {0}; @@ -1241,6 +1267,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SArray* pDelList = pBlockScanInfo->delSkyline; bool freeTSRow = false; + uint64_t uid = pBlockScanInfo->uid; // ascending order traverse if (ASCENDING_TRAVERSE(pReader->order)) { @@ -1276,9 +1303,9 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerGetRow(&merge, &pTSRow); } } else { // descending order: mem rows -----> imem rows ------> file block - updateSchema(pRow, pBlockScanInfo->uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pReader->pSchema); + tRowMergerInit(&merge, pRow, pSchema); doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, &fRow); @@ -1289,7 +1316,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } tRowMergerClear(&merge); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; @@ -1333,7 +1360,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* } tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); return TSDB_CODE_SUCCESS; } else { // key > ik.ts || key > k.ts ASSERT(key != ik.ts); @@ -1342,7 +1369,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [4] ik.ts < k.ts <= key if (ik.ts < k.ts) { doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -1353,7 +1380,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [6] k.ts < ik.ts <= key if (k.ts < ik.ts) { doMergeMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -1365,7 +1392,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* ASSERT(key > ik.ts && key > k.ts); doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } @@ -1373,9 +1400,9 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { // descending order scan // [1/2] k.ts >= ik.ts && k.ts >= key if (k.ts >= ik.ts && k.ts >= key) { - updateSchema(pRow, uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pReader->pSchema); + tRowMergerInit(&merge, pRow, pSchema); doMergeRowsInBuf(&pBlockScanInfo->iter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader); if (ik.ts == k.ts) { @@ -1390,7 +1417,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* } tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); return TSDB_CODE_SUCCESS; } else { ASSERT(ik.ts != k.ts); // this case has been included in the previous if branch @@ -1399,7 +1426,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [4] ik.ts > key >= k.ts if (ik.ts > key) { doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -1414,7 +1441,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } @@ -1427,7 +1454,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; @@ -1495,7 +1522,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); taosMemoryFree(pTSRow); tRowMergerClear(&merge); @@ -2146,6 +2173,7 @@ TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pRea } } + int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader) { while (1) { @@ -2166,22 +2194,8 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe break; } - int32_t sversion = TSDBROW_SVERSION(pRow); - STSchema* pTSchema = NULL; - if (pReader->pSchema == NULL || sversion != pReader->pSchema->version) { - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pTSchema); - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } - } else { - pTSchema = pReader->pSchema; - } - + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); tRowMergerAdd(pMerger, pRow, pTSchema); - - if (pTSchema != pReader->pSchema) { - taosMemoryFree(pTSchema); - } } return TSDB_CODE_SUCCESS; @@ -2289,21 +2303,13 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc return TSDB_CODE_SUCCESS; } -void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader) { - int32_t sversion = TSDBROW_SVERSION(pRow); - - if (pReader->pSchema == NULL) { - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pSchema); - } else if (pReader->pSchema->version != sversion) { - taosMemoryFreeClear(pReader->pSchema); - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pSchema); - } -} - void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow) { + + TSDBROW* pNextRow = NULL; + TSDBROW current = *pRow; + { // if the timestamp of the next valid row has a different ts, return current row directly - TSDBROW current = *pRow; pIter->hasVal = tsdbTbDataIterNext(pIter->iter); if (!pIter->hasVal) { @@ -2311,14 +2317,14 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe *freeTSRow = false; return; } else { // has next point in mem/imem - TSDBROW* pNextRow = getValidRow(pIter, pDelList, pReader); + pNextRow = getValidRow(pIter, pDelList, pReader); if (pNextRow == NULL) { *pTSRow = current.pTSRow; *freeTSRow = false; return; } - if (TSDBROW_KEY(¤t).ts != TSDBROW_KEY(pNextRow).ts) { + if (current.pTSRow->ts != pNextRow->pTSRow->ts) { *pTSRow = current.pTSRow; *freeTSRow = false; return; @@ -2327,30 +2333,20 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe } SRowMerger merge = {0}; - TSDBKEY k = TSDBROW_KEY(pRow); // get the correct schema for data in memory - int32_t sversion = TSDBROW_SVERSION(pRow); - STSchema* pTSchema = NULL; - if (pReader->pSchema == NULL || sversion != pReader->pSchema->version) { - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pTSchema); - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } - } else { - pTSchema = pReader->pSchema; - } + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); - tRowMergerInit2(&merge, pReader->pSchema, pRow, pTSchema); - doMergeRowsInBuf(pIter, uid, k.ts, pDelList, &merge, pReader); + tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + + STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + tRowMergerAdd(&merge, pNextRow, pTSchema1); + + doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); tRowMergerGetRow(&merge, pTSRow); tRowMergerClear(&merge); *freeTSRow = true; - - if (sversion != pReader->pSchema->version) { - taosMemoryFree(pTSchema); - } } void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, @@ -2361,17 +2357,17 @@ void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlo TSDBKEY ik = TSDBROW_KEY(piRow); if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem - updateSchema(piRow, pBlockScanInfo->uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pReader->pSchema); + tRowMergerInit(&merge, piRow, pSchema); doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, pRow); doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } else { - updateSchema(pRow, pBlockScanInfo->uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pReader->pSchema); + tRowMergerInit(&merge, pRow, pSchema); doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, piRow); @@ -2432,12 +2428,12 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow) { +int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid) { int32_t numOfRows = pBlock->info.rows; int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - STSchema* pSchema = pReader->pSchema; + STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); SColVal colVal = {0}; int32_t i = 0, j = 0; @@ -2453,7 +2449,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* col_id_t colId = pColInfoData->info.colId; if (colId == pSchema->columns[j].colId) { - tTSRowGetVal(pTSRow, pReader->pSchema, j, &colVal); + tTSRowGetVal(pTSRow, pSchema, j, &colVal); doCopyColVal(pColInfoData, numOfRows, i, &colVal, pSupInfo); i += 1; j += 1; @@ -2529,7 +2525,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e break; } - doAppendRowFromTSRow(pBlock, pReader, pTSRow); + doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo->uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -2755,6 +2751,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); + taosMemoryFree(pReader->pMemSchema); taosMemoryFreeClear(pReader); } From f13215a9016adc902b97a7ff0b9f2e05af9623f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 10:17:48 +0800 Subject: [PATCH 43/96] test:fix invalid test cases. --- tests/script/tsim/parser/insert_multiTbl.sim | 2 +- tests/script/tsim/parser/last_cache_query.sim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/tsim/parser/insert_multiTbl.sim b/tests/script/tsim/parser/insert_multiTbl.sim index 78d3352378..fb6da1cfba 100644 --- a/tests/script/tsim/parser/insert_multiTbl.sim +++ b/tests/script/tsim/parser/insert_multiTbl.sim @@ -49,7 +49,7 @@ if $rows != 12 then return -1 endi if $data02 != NULL then - print "expect: NULL, act: $data03" + print "expect: NULL, act: $data02" return -1 endi if $data03 != @binary00@ then diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim index 7bafe82f5d..ebbb784da5 100644 --- a/tests/script/tsim/parser/last_cache_query.sim +++ b/tests/script/tsim/parser/last_cache_query.sim @@ -304,8 +304,8 @@ endi if $data21 != 24 then return -1 endi -if $data22 != 11.000000000 then - print $data02 +if $data22 != 11.000000000 then + print expect 11.000000000 actual: $data22 return -1 endi if $data23 != 25 then From 29c84833d3d7916d1ef83418e7fb1931d7b537e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 10:34:01 +0800 Subject: [PATCH 44/96] fix(query):fix invalid free. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index e1e236bf11..714af80e83 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1279,6 +1279,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } else if (k.ts < key) { // k.ts < key doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader, &freeTSRow); @@ -1290,6 +1291,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } else { // descending order scan if (key < k.ts) { @@ -1301,6 +1303,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } else { // descending order: mem rows -----> imem rows ------> file block STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); @@ -1312,13 +1315,17 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } tRowMergerClear(&merge); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - taosMemoryFree(pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } + return TSDB_CODE_SUCCESS; } From 43a67f6da9b8c990a03cbb93be5f20fca7f15487 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 10:57:44 +0800 Subject: [PATCH 45/96] fix: insert error of unbound columns --- source/libs/parser/src/parInsert.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 6e47e26a7a..4756f65bf5 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -18,10 +18,10 @@ #include "parInt.h" #include "parToken.h" #include "parUtil.h" +#include "query.h" #include "tglobal.h" #include "ttime.h" #include "ttypes.h" -#include "query.h" #define NEXT_TOKEN(pSql, sToken) \ do { \ @@ -1189,12 +1189,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, if (!isParseBindParam) { // set the null value for the columns that do not assign values if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(schema[i].type), true, schema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } *gotRow = true; @@ -1267,7 +1262,8 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) { - return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than INT32_MAX"); + return buildInvalidOperationMsg(&pCxt->msg, + "too many rows in sql, total number of rows should be less than INT32_MAX"); } dataBuf->numOfTables = 1; @@ -1339,7 +1335,8 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STa SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) { - return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than INT32_MAX"); + return buildInvalidOperationMsg(&pCxt->msg, + "too many rows in sql, total number of rows should be less than INT32_MAX"); } dataBuf->numOfTables = 1; @@ -2060,7 +2057,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu } } } - if(rowEnd) { + if (rowEnd) { tdSRowEnd(pBuilder); } #ifdef TD_DEBUG_PRINT_ROW @@ -2077,7 +2074,8 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { - return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than INT32_MAX"); + return buildInvalidOperationMsg(&pBuf, + "too many rows in sql, total number of rows should be less than INT32_MAX"); } } From a2a1da06dd056b58d040fb83957a239377b9520b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 13:45:44 +0800 Subject: [PATCH 46/96] other:merge 3.0 --- contrib/CMakeLists.txt | 17 +++++++ include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 57 ++++++++++++++--------- source/dnode/vnode/src/tq/tqExec.c | 13 +++++- source/libs/executor/src/dataDispatcher.c | 8 ++-- source/libs/executor/src/executor.c | 33 +++++++++---- source/libs/qworker/src/qworker.c | 29 ++++++------ source/libs/stream/src/streamExec.c | 23 ++++++--- 8 files changed, 124 insertions(+), 58 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b4e8825431..de7b75a245 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -27,6 +27,10 @@ else () cat("${TD_SUPPORT_DIR}/taosadapter_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() +if(TD_LINUX_64 AND JEMALLOC_ENABLED) + cat("${TD_SUPPORT_DIR}/jemalloc_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + # pthread if(${BUILD_PTHREAD}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -392,6 +396,19 @@ if(${BUILD_WITH_SQLITE}) endif(NOT TD_WINDOWS) endif(${BUILD_WITH_SQLITE}) +# jemalloc +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + include(ExternalProject) + ExternalProject_Add(jemalloc + PREFIX "jemalloc" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ + BUILD_COMMAND ${MAKE} + ) + INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include) +ENDIF () + # addr2line if(${BUILD_ADDR2LINE}) if(NOT ${TD_WINDOWS}) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 7bc2ede4fc..a3cc7aaa8a 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -123,7 +123,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table * @param handle * @return */ -int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds); +int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds); /** * kill the ongoing query and free the query handle and corresponding resources automatically diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 662558529d..ef9a64e946 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -604,28 +604,44 @@ _end: static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, int8_t blkType) { - while (1) { - SSDataBlock *output = NULL; - uint64_t ts; + SArray *pResList = taosArrayInit(1, POINTER_BYTES); + if (pResList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } - int32_t code = qExecTask(taskInfo, &output, &ts); + while (1) { + uint64_t ts; + int32_t code = qExecTask(taskInfo, pResList, &ts); if (code < 0) { smaError("vgId:%d, qExecTask for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr(code)); goto _err; } - if (output) { -#if 0 + if (taosArrayGetSize(pResList) == 0) { + if (terrno == 0) { + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); + } else { + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, terrstr()); + goto _err; + } + + break; + } + + for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { + SSDataBlock *output = taosArrayGetP(pResList, i); + +#if 1 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, pItem->level); - SArray *pResult = taosArrayInit(1, sizeof(SSDataBlock)); - taosArrayPush(pResult, output); - blockDebugShowDataBlocks(pResult, flag); - taosArrayDestroy(pResult); +// blockDebugShowDataBlocks(output, flag); +// taosArrayDestroy(pResult); #endif - STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); + STsdb * sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); SSubmitReq *pReq = NULL; + // TODO: the schema update should be handled later(TD-17965) if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, SMA_VID(pSma), suid) < 0) { smaError("vgId:%d, build submit req for rsma stable %" PRIi64 " level %" PRIi8 " failed since %s", @@ -644,17 +660,14 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm SMA_VID(pSma), suid, pItem->level, output->info.version); taosMemoryFreeClear(pReq); - } else if (terrno == 0) { - smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); - break; - } else { - smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, terrstr()); - goto _err; } } + taosArrayDestroy(pResList); return TSDB_CODE_SUCCESS; + _err: + taosArrayDestroy(pResList); return TSDB_CODE_FAILED; } @@ -1407,7 +1420,7 @@ int32_t tdRSmaFetchSend(SSma *pSma, SRSmaInfo *pInfo, int8_t level) { tEncoderClear(&encoder); } tEncoderClear(&encoder); - + ((SMsgHead *)pBuf)->vgId = SMA_VID(pSma); ((SMsgHead *)pBuf)->contLen = contLen + sizeof(SMsgHead); @@ -1434,10 +1447,10 @@ _err: /** * @brief fetch rsma data of level 2/3 and submit - * - * @param pSma - * @param pMsg - * @return int32_t + * + * @param pSma + * @param pMsg + * @return int32_t */ int32_t smaProcessFetch(SSma *pSma, void *pMsg) { SRpcMsg *pRpcMsg = (SRpcMsg *)pMsg; diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 435bbb77b8..09a5f1a8c6 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -80,14 +80,22 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa } int32_t rowCnt = 0; + SArray* pResList = taosArrayInit(4, POINTER_BYTES); + while (1) { + taosArrayClear(pResList); + SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; tqDebug("task start to execute"); - if (qExecTask(task, &pDataBlock, &ts) < 0) { + if (qExecTask(task, pResList, &ts) < 0) { ASSERT(0); } - tqDebug("task execute end, get %p", pDataBlock); + + if (taosArrayGetSize(pResList) > 0) { + pDataBlock = taosArrayGet(pResList, 0); + tqDebug("task execute end, get %p", pDataBlock); + } if (pDataBlock != NULL) { if (pRsp->withTbName) { @@ -143,6 +151,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa break; } + taosArrayDestroy(pResList); return 0; } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 3a37b5e760..977adcaa5e 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -69,10 +69,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { // clang-format off // data format: -// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | total length | numOfRows | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) -// | |sizeof(int32) | sizeof(int32)| sizeof(uint64_t) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | -// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// +----------------+------------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// |SDataCacheEntry | version | total length | numOfRows | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) +// | | sizeof(int32_t) |sizeof(int32) | sizeof(int32)| sizeof(uint64_t) | (sizeof(int8_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | +// +----------------+------------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header // clang-format on diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 897ddb59ec..5bbf5928da 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -422,11 +422,17 @@ int waitMoment(SQInfo* pQInfo) { } #endif -int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { +static void freeBlock(void* param) { + SSDataBlock* pBlock = *(SSDataBlock**) param; + blockDataDestroy(pBlock); +} + +int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); - *pRes = NULL; + taosArrayClearEx(pResList, freeBlock); + int64_t curOwner = 0; if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); @@ -457,23 +463,34 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); + int32_t current = 0; + SSDataBlock* pRes = NULL; + int64_t st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); + while((pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot)) != NULL) { + SSDataBlock* p = createOneDataBlock(pRes, true); + current += p->info.rows; + ASSERT(p->info.rows > 0); + taosArrayPush(pResList, &p); + + if (current >= 4096) { + break; + } + } + uint64_t el = (taosGetTimestampUs() - st); pTaskInfo->cost.elapsedTime += el; - if (NULL == *pRes) { + if (NULL == pRes) { *useconds = pTaskInfo->cost.elapsedTime; } cleanUpUdfs(); - - int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; - qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", - GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0); + qDebug("%s task suspended, %d rows in %d blocks returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", + GET_TASKID(pTaskInfo), current, (int32_t) taosArrayGetSize(pResList), total, 0, el / 1000.0); atomic_store_64(&pTaskInfo->owner, 0); return pTaskInfo->code; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 7c5cfda81f..85a1b0444c 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -75,22 +75,20 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { int32_t code = 0; bool qcontinue = true; - SSDataBlock *pRes = NULL; uint64_t useconds = 0; int32_t i = 0; int32_t execNum = 0; qTaskInfo_t taskHandle = ctx->taskHandle; DataSinkHandle sinkHandle = ctx->sinkHandle; + SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (true) { QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); - pRes = NULL; - // if *taskHandle is NULL, it's killed right now if (taskHandle) { qwDbgSimulateSleep(); - code = qExecTask(taskHandle, &pRes, &useconds); + code = qExecTask(taskHandle, pResList, &useconds); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -103,9 +101,8 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { ++execNum; - if (NULL == pRes) { + if (taosArrayGetSize(pResList) == 0) { QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); - dsEndPut(sinkHandle, useconds); QW_ERR_RET(qwHandleTaskComplete(QW_FPARAMS(), ctx)); @@ -117,19 +114,20 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { break; } - int32_t rows = pRes->info.rows; + for(int32_t j = 0; j < taosArrayGetSize(pResList); ++j) { + SSDataBlock *pRes = taosArrayGetP(pResList, j); + ASSERT(pRes->info.rows > 0); - ASSERT(pRes->info.rows > 0); + SInputData inputData = {.pData = pRes}; + code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); + if (code) { + QW_TASK_ELOG("dsPutDataBlock failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); + } - SInputData inputData = {.pData = pRes}; - code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); - if (code) { - QW_TASK_ELOG("dsPutDataBlock failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); + QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); } - QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", rows, qcontinue); - if (!qcontinue) { if (queryStop) { *queryStop = true; @@ -151,6 +149,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { } } + taosArrayDestroy(pResList); QW_RET(code); } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 7512f792c1..b92f658de0 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -43,13 +43,17 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* } // exec + SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SSDataBlock* output = NULL; uint64_t ts = 0; - if (qExecTask(exec, &output, &ts) < 0) { + + taosArrayClear(pResList); + if (qExecTask(exec, pResList, &ts) < 0) { ASSERT(false); } - if (output == NULL) { + + if (taosArrayGetSize(pResList) == 0) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SSDataBlock block = {0}; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data; @@ -65,6 +69,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* break; } + output = taosArrayGetP(pResList, 0); if (output->info.type == STREAM_RETRIEVE) { if (streamBroadcastToChildren(pTask, output) < 0) { // TODO @@ -79,6 +84,8 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); } + + taosArrayDestroy(pResList); return 0; } @@ -98,6 +105,7 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { void* exec = pTask->exec.executor; + SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { @@ -107,14 +115,17 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { int32_t batchCnt = 0; while (1) { - SSDataBlock* output = NULL; - uint64_t ts = 0; - if (qExecTask(exec, &output, &ts) < 0) { + uint64_t ts = 0; + taosArrayClear(pResList); + if (qExecTask(exec, pResList, &ts) < 0) { ASSERT(0); } - if (output == NULL) break; + + if (taosArrayGetSize(pResList) == 0) break; SSDataBlock block = {0}; + SSDataBlock* output = taosArrayGetP(pResList, 0); + assignOneDataBlock(&block, output); block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); From b9c6e0cdf9db43c6b56b3cf7ec7757ea2ee79843 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 13:55:51 +0800 Subject: [PATCH 47/96] refactor: stream invoke previous APIs. --- include/libs/executor/executor.h | 3 +- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/tq/tqExec.c | 13 +----- source/libs/executor/src/executor.c | 59 +++++++++++++++++++++++++- source/libs/qworker/src/qworker.c | 2 +- source/libs/stream/src/streamExec.c | 23 +++------- 6 files changed, 70 insertions(+), 32 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index a3cc7aaa8a..a64815f14f 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -123,7 +123,8 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table * @param handle * @return */ -int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds); +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds); +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds); /** * kill the ongoing query and free the query handle and corresponding resources automatically diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ef9a64e946..6b513f0242 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -612,7 +612,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm while (1) { uint64_t ts; - int32_t code = qExecTask(taskInfo, pResList, &ts); + int32_t code = qExecTaskOpt(taskInfo, pResList, &ts); if (code < 0) { smaError("vgId:%d, qExecTask for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr(code)); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 09a5f1a8c6..435bbb77b8 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -80,22 +80,14 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa } int32_t rowCnt = 0; - SArray* pResList = taosArrayInit(4, POINTER_BYTES); - while (1) { - taosArrayClear(pResList); - SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; tqDebug("task start to execute"); - if (qExecTask(task, pResList, &ts) < 0) { + if (qExecTask(task, &pDataBlock, &ts) < 0) { ASSERT(0); } - - if (taosArrayGetSize(pResList) > 0) { - pDataBlock = taosArrayGet(pResList, 0); - tqDebug("task execute end, get %p", pDataBlock); - } + tqDebug("task execute end, get %p", pDataBlock); if (pDataBlock != NULL) { if (pRsp->withTbName) { @@ -151,7 +143,6 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa break; } - taosArrayDestroy(pResList); return 0; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 5bbf5928da..328a65bec4 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -427,7 +427,7 @@ static void freeBlock(void* param) { blockDataDestroy(pBlock); } -int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); @@ -496,6 +496,63 @@ int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { return pTaskInfo->code; } +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + int64_t threadId = taosGetSelfPthreadId(); + + *pRes = NULL; + int64_t curOwner = 0; + if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { + qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); + pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; + return pTaskInfo->code; + } + + if (pTaskInfo->cost.start == 0) { + pTaskInfo->cost.start = taosGetTimestampMs(); + } + + if (isTaskKilled(pTaskInfo)) { + atomic_store_64(&pTaskInfo->owner, 0); + qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); + return TSDB_CODE_SUCCESS; + } + + // error occurs, record the error code and return to client + int32_t ret = setjmp(pTaskInfo->env); + if (ret != TSDB_CODE_SUCCESS) { + pTaskInfo->code = ret; + cleanUpUdfs(); + qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); + atomic_store_64(&pTaskInfo->owner, 0); + + return pTaskInfo->code; + } + + qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); + + int64_t st = taosGetTimestampUs(); + + *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); + uint64_t el = (taosGetTimestampUs() - st); + + pTaskInfo->cost.elapsedTime += el; + if (NULL == *pRes) { + *useconds = pTaskInfo->cost.elapsedTime; + } + + cleanUpUdfs(); + + int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; + uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; + + qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", + GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0); + + atomic_store_64(&pTaskInfo->owner, 0); + return pTaskInfo->code; +} + int32_t qKillTask(qTaskInfo_t qinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; if (pTaskInfo == NULL) { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 85a1b0444c..e06b752862 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -88,7 +88,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { // if *taskHandle is NULL, it's killed right now if (taskHandle) { qwDbgSimulateSleep(); - code = qExecTask(taskHandle, pResList, &useconds); + code = qExecTaskOpt(taskHandle, pResList, &useconds); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index b92f658de0..7512f792c1 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -43,17 +43,13 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* } // exec - SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SSDataBlock* output = NULL; uint64_t ts = 0; - - taosArrayClear(pResList); - if (qExecTask(exec, pResList, &ts) < 0) { + if (qExecTask(exec, &output, &ts) < 0) { ASSERT(false); } - - if (taosArrayGetSize(pResList) == 0) { + if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SSDataBlock block = {0}; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data; @@ -69,7 +65,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* break; } - output = taosArrayGetP(pResList, 0); if (output->info.type == STREAM_RETRIEVE) { if (streamBroadcastToChildren(pTask, output) < 0) { // TODO @@ -84,8 +79,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); } - - taosArrayDestroy(pResList); return 0; } @@ -105,7 +98,6 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { void* exec = pTask->exec.executor; - SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { @@ -115,17 +107,14 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { int32_t batchCnt = 0; while (1) { - uint64_t ts = 0; - taosArrayClear(pResList); - if (qExecTask(exec, pResList, &ts) < 0) { + SSDataBlock* output = NULL; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { ASSERT(0); } - - if (taosArrayGetSize(pResList) == 0) break; + if (output == NULL) break; SSDataBlock block = {0}; - SSDataBlock* output = taosArrayGetP(pResList, 0); - assignOneDataBlock(&block, output); block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); From 9b1201a61ae00d5ca919d11dbda4aabf74b36258 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 15:18:21 +0800 Subject: [PATCH 48/96] fix: slimit error --- source/libs/planner/src/planLogicCreater.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 12646c773c..875ec21b24 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -469,7 +469,8 @@ static int32_t createGroupKeysFromPartKeys(SNodeList* pPartKeys, SNodeList** pOu } static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { - return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; + return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP + : GROUP_ACTION_NONE; } static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { From dbf04b46dfd23a3c45b56f635daad46c3881fa5c Mon Sep 17 00:00:00 2001 From: Yang Zhao Date: Tue, 9 Aug 2022 15:34:50 +0800 Subject: [PATCH 49/96] docs: udpate python connector doc for 3.0 --- docs/zh/14-reference/03-connector/python.mdx | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/docs/zh/14-reference/03-connector/python.mdx b/docs/zh/14-reference/03-connector/python.mdx index a77dc71db7..d7b17dc74a 100644 --- a/docs/zh/14-reference/03-connector/python.mdx +++ b/docs/zh/14-reference/03-connector/python.mdx @@ -306,8 +306,7 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 | [bind_row.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-row.py) | 参数绑定，一次绑定一行 | | [insert_lines.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/insert-lines.py) | InfluxDB 行协议写入 | | [json_tag.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/json-tag.py) | 使用 JSON 类型的标签 | -| [subscribe-async.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/subscribe-async.py) | 异步订阅 | -| [subscribe-sync.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/subscribe-sync.py) | 同步订阅 | +| [tmq.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq.py) | tmq 订阅 | ## 其它说明 @@ -326,23 +325,15 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 1. https://stackoverflow.com/questions/10611328/parsing-datetime-strings-containing-nanoseconds 2. https://www.python.org/dev/peps/pep-0564/ - -## 常见问题 - -欢迎[提问或报告问题](https://github.com/taosdata/taos-connector-python/issues)。 - ## 重要更新 -| 连接器版本 | 重要更新 | 发布日期 | -| ---------- | --------------------------------------------------------------------------------- | ---------- | -| 2.3.1 | 1. support TDengine REST API
2. remove support for Python version below 3.6 | 2022-04-28 | -| 2.2.5 | support timezone option when connect | 2022-04-13 | -| 2.2.2 | support sqlalchemy dialect plugin | 2022-03-28 | - - [**Release Notes**](https://github.com/taosdata/taos-connector-python/releases) ## API 参考 - [taos](https://docs.taosdata.com/api/taospy/taos/) - [taosrest](https://docs.taosdata.com/api/taospy/taosrest) + +## 常见问题 + +欢迎[提问或报告问题](https://github.com/taosdata/taos-connector-python/issues)。 From 9e498a7859ea354e82cc13caa4035e85d3ce0003 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 9 Aug 2022 15:40:39 +0800 Subject: [PATCH 50/96] fix: set none for unbound cols --- source/libs/parser/src/parInsert.c | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 4756f65bf5..f2d0ede486 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1966,12 +1966,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in } // set the null value for the columns that do not assign values if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } tdSRowEnd(pBuilder); #ifdef TD_DEBUG_PRINT_ROW @@ -2050,12 +2045,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu // set the null value for the columns that do not assign values if (rowEnd && (spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } if (rowEnd) { tdSRowEnd(pBuilder); @@ -2432,12 +2422,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols // set the null value for the columns that do not assign values if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } tdSRowEnd(pBuilder); From 41d2c922311542eb0a393da065df3d0c52229f50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 16:54:05 +0800 Subject: [PATCH 51/96] fix(query): copy null value. --- source/common/src/tdatablock.c | 4 ---- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9a1d761e0d..dba30bb876 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1329,10 +1329,6 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); - if (pSrc->pData == NULL) { - continue; - } - colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 714af80e83..37861a0e0a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -182,6 +182,7 @@ static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes); + tsdbInfo("-------------------%d\n", pCol->info.bytes); } } @@ -674,6 +675,7 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ colDataAppendNULL(pColInfoData, rowIndex); } else { varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); + ASSERT(pColVal->value.nData <= pColInfoData->info.bytes); memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); colDataAppend(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); } From 30b51e4041f696c512af0610c4f0000c56b72864 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 9 Aug 2022 16:57:41 +0800 Subject: [PATCH 52/96] enh: row optimization --- include/common/tdataformat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 311eb72b4d..a304dd93b2 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -200,8 +200,8 @@ struct STag { #if 1 //================================================================================================================================================ // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP -#define TD_SUPPORT_READ2 -#define TD_SUPPORT_BACK2 // suppport back compatibility of 2.0 +#undef TD_SUPPORT_READ2 +#undef TD_SUPPORT_BACK2 // suppport back compatibility of 2.0 #define TASSERT(x) ASSERT(x) From 8b86d7342d89be79b32dd70a076e14cdad6fbdd5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 17:21:31 +0800 Subject: [PATCH 53/96] fix(query): check the sma load status, and return if succeed. --- source/libs/executor/src/scanoperator.c | 10 +++++++--- tests/script/tsim/testsuit.sim | 14 +++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b2fb80b68f..e401a1bd96 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -312,9 +312,13 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; - loadSMA = true; // mark the operator of load sma; + loadSMA = true; // mark the operation of load sma; bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); - if (!success) { // failed to load the block sma data, data block statistics does not exist, load data block instead + if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead + qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + return TSDB_CODE_SUCCESS; + } else { *status = FUNC_DATA_REQUIRED_DATA_LOAD; } } @@ -339,7 +343,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } } - // try to filter datablock according to current results + // try to filter data block according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index e0c46ae5fe..89a4babb0b 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -1,10 +1,10 @@ -run tsim/user/password.sim -run tsim/user/privilege_db.sim -run tsim/user/privilege_sysinfo.sim -run tsim/user/basic.sim -run tsim/table/basic1.sim -run tsim/trans/lossdata1.sim -run tsim/trans/create_db.sim +#run tsim/user/password.sim +#run tsim/user/privilege_db.sim +#run tsim/user/privilege_sysinfo.sim +#run tsim/user/basic.sim +#run tsim/table/basic1.sim +#run tsim/trans/lossdata1.sim +#run tsim/trans/create_db.sim run tsim/stable/alter_metrics.sim run tsim/stable/tag_modify.sim run tsim/stable/alter_comment.sim From bddb3e356d06def42f7430caeef971007f425d80 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 17:55:00 +0800 Subject: [PATCH 54/96] fix(query): expand the sink buffer. --- source/libs/executor/src/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 328a65bec4..497681ded1 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -353,7 +353,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, goto _error; } - SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100}; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 10000, .maxDataBlockNumPerQuery = 5000}; code = dsDataSinkMgtInit(&cfg); if (code != TSDB_CODE_SUCCESS) { goto _error; From d680da33806a3b0e3fbaac48a3dd01cbcfd73721 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 18:35:10 +0800 Subject: [PATCH 55/96] fix(query): fix sma info in query procesing. --- source/libs/executor/src/executor.c | 4 ++++ source/libs/executor/src/scanoperator.c | 3 +++ tests/script/tsim/parser/limit1_tb.sim | 4 ++++ tests/script/tsim/parser/mixed_blocks.sim | 1 + 4 files changed, 12 insertions(+) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 497681ded1..1fc0f5ff51 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -348,6 +348,8 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, taosThreadOnce(&initPoolOnce, initRefPool); atexit(cleanupRefPool); + qDebug("start to create subplan task, TID:0x%"PRIx64 " QID:0x%"PRIx64, taskId, pSubplan->id.queryId); + int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -372,6 +374,8 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } } + qDebug("subplan task create completed, TID:0x%"PRIx64 " QID:0x%"PRIx64, taskId, pSubplan->id.queryId); + _error: // if failed to add ref for all tables in this query, abort current query return code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e401a1bd96..a0f2560e7b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -343,6 +343,9 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } } + // free the sma info, since it should not be involved in later computing process. + taosMemoryFreeClear(pBlock->pBlockAgg); + // try to filter data block according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { diff --git a/tests/script/tsim/parser/limit1_tb.sim b/tests/script/tsim/parser/limit1_tb.sim index 1ef6a62291..36c91fe4c7 100644 --- a/tests/script/tsim/parser/limit1_tb.sim +++ b/tests/script/tsim/parser/limit1_tb.sim @@ -263,6 +263,9 @@ sql select sum(c1), avg(c2), stddev(c3), max(c4), min(c5), count(c6), first(c7), if $rows != 0 then return -1 endi + +print select sum(c1), avg(c2), stddev(c3), max(c4), min(c5), count(c6), first(c7), last(c8), last(c9) from $tb where ts >= $ts0 and ts <= $tsu and c1>1 and c2<9 and c3>2 and c4<8 and c5>4 and c6<6 limit 1 offset 0 + sql select sum(c1), avg(c2), stddev(c3), max(c4), min(c5), count(c6), first(c7), last(c8), last(c9) from $tb where ts >= $ts0 and ts <= $tsu and c1>1 and c2<9 and c3>2 and c4<8 and c5>4 and c6<6 limit 1 offset 0 if $rows != 1 then return -1 @@ -270,6 +273,7 @@ endi $val = $rowNum / 10 $val = $val * 5 if $data00 != $val then + print $data00, expect $val return -1 endi if $data01 != 5.000000000 then diff --git a/tests/script/tsim/parser/mixed_blocks.sim b/tests/script/tsim/parser/mixed_blocks.sim index 76ac7c1c54..80cdfcac99 100644 --- a/tests/script/tsim/parser/mixed_blocks.sim +++ b/tests/script/tsim/parser/mixed_blocks.sim @@ -92,6 +92,7 @@ if $data03 != 319 then return -1 endi if $data04 != 318.000000000 then + print expect 318.000000000, actual: $data04 return -1 endi if $data05 != 1 then From 7b70f54a6e3c30accbd4e3fe50d5500f1b812b4e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 18:47:23 +0800 Subject: [PATCH 56/96] test: add log. --- source/libs/executor/src/scanoperator.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a0f2560e7b..a211542de1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2665,11 +2665,17 @@ _error: int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idStr) { + + int64_t st = taosGetTimestampUs(); + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { return code; } + int64_t st1 = taosGetTimestampUs(); + qDebug("generate queried table list completed, elapsed time:%.2f ms %s", (st1-st)/1000.0, idStr); + if (taosArrayGetSize(pTableListInfo->pTableList) == 0) { qDebug("no table qualified for query, %s" PRIx64, idStr); return TSDB_CODE_SUCCESS; @@ -2681,6 +2687,9 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return code; } + int64_t st2 = taosGetTimestampUs(); + qDebug("generate group id map completed, elapsed time:%.2f ms %s", (st2-st1)/1000.0, idStr); + return TSDB_CODE_SUCCESS; } From 3e626d56d15978b39ff3ea835fe21593719aa97c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 9 Aug 2022 20:32:02 +0800 Subject: [PATCH 57/96] fix: set none if cols not full --- source/libs/executor/src/dataInserter.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 8e355ad3df..26e1f0a3ee 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -213,6 +213,9 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); } } + if(!fullCol) { + rb.hasNone = true; + } tdSRowEnd(&rb); if (ignoreRow) { From 7df0a65e75e427d1a0df06c6b2929e48ad072d75 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 20:57:34 +0800 Subject: [PATCH 58/96] fix(query): set correct meta size in handling json. --- source/client/src/clientImpl.c | 12 ++++++++++-- tests/script/tsim/compute/diff2.sim | 1 - 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7838d87a20..d3079a48d7 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1571,10 +1571,18 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int return TSDB_CODE_SUCCESS; } +static int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { + int32_t cols = *(int32_t*) (p + sizeof(int32_t) * 3); + ASSERT(numOfCols == cols); + + return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t)*3 + sizeof(uint64_t) + numOfCols * (sizeof(int8_t) + sizeof(int32_t)); +} + static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { char* p = (char*)pResultInfo->pData; - int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + // version + length + numOfRows + numOfCol + groupId + flag_segment + column_info + int32_t len = getVersion1BlockMetaSize(p, numOfCols); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; @@ -1642,7 +1650,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* p1 = pResultInfo->convertJson; int32_t totalLen = 0; - int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + int32_t len = getVersion1BlockMetaSize(p, numOfCols); memcpy(p1, p, len); p += len; diff --git a/tests/script/tsim/compute/diff2.sim b/tests/script/tsim/compute/diff2.sim index a09bee991e..5a2c3706ee 100644 --- a/tests/script/tsim/compute/diff2.sim +++ b/tests/script/tsim/compute/diff2.sim @@ -78,7 +78,6 @@ endi sql select diff(c7) from $tb sql_error select diff(c8) from $tb sql_error select diff(c9) from $tb -sql_error select diff(ts) from $tb sql select diff(c1), diff(c2) from $tb sql select 2+diff(c1) from $tb From ae7b2954f4918c5f6e7fe6bf07b91c706fa688b1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 21:02:24 +0800 Subject: [PATCH 59/96] test: remove invalid cases. --- tests/system-test/2-query/diff.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index 76d4891a1c..ceef3f93eb 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -83,9 +83,6 @@ class TDTestCase: tdSql.execute(f"insert into {dbname}.stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - tdSql.error(f"select diff(ts) from {dbname}.stb") - tdSql.error(f"select diff(ts) from {dbname}.stb_1") - # tdSql.error(f"select diff(col7) from {dbname}.stb") tdSql.error(f"select diff(col8) from {dbname}.stb") From 597d360d853c1479c76a224a1daee239a6da5314 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 21:31:22 +0800 Subject: [PATCH 60/96] test: remove invalid cases. --- tests/system-test/2-query/function_diff.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/system-test/2-query/function_diff.py b/tests/system-test/2-query/function_diff.py index 2f463e59a0..492703d3ec 100644 --- a/tests/system-test/2-query/function_diff.py +++ b/tests/system-test/2-query/function_diff.py @@ -256,8 +256,6 @@ class TDTestCase: tdSql.error(self.diff_query_form(col='c%')) # col is spercial char 1 tdSql.error(self.diff_query_form(col='c_')) # col is spercial char 2 tdSql.error(self.diff_query_form(col='c.')) # col is spercial char 3 - tdSql.error(self.diff_query_form(col='c3')) # timestamp col - tdSql.error(self.diff_query_form(col='ts')) # Primary key tdSql.error(self.diff_query_form(col='avg(c1)')) # expr col # tdSql.error(self.diff_query_form(col='c6')) # bool col tdSql.query("select diff(c6) from t1") From abc90650944c782b23851f09fac1c7b4f81e94d0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 13:13:25 +0800 Subject: [PATCH 61/96] fix(query): set correct raw block info. --- source/client/inc/clientInt.h | 6 ++---- source/client/src/clientImpl.c | 2 +- source/client/src/tmq.c | 2 +- source/libs/executor/src/executorimpl.c | 1 - source/libs/executor/src/sortoperator.c | 2 +- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index e7a099cf23..f275ae0885 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -101,10 +101,6 @@ typedef struct SQueryExecMetric { int64_t rsp; // receive response from server, us } SQueryExecMetric; -typedef struct SHeartBeatInfo { - void* pTimer; // timer, used to send request msg to mnode -} SHeartBeatInfo; - struct SAppInstInfo { int64_t numOfConns; SCorEpSet mgmtEp; @@ -256,6 +252,8 @@ SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool valida TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly); void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); +int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols); + static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; return (SReqResultInfo*)&msg->resInfo; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d3079a48d7..5c37e6dc15 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1571,7 +1571,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int return TSDB_CODE_SUCCESS; } -static int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { +int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { int32_t cols = *(int32_t*) (p + sizeof(int32_t) * 3); ASSERT(numOfCols == cols); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index e2619a78b1..8f866bde8e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -2947,7 +2947,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; - char* pStart = pData + sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols) - numOfCols * (sizeof(int8_t) + sizeof(int32_t)); int32_t* colLength = (int32_t*)pStart; pStart += sizeof(int32_t) * numOfCols; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5f5c035523..d3ced810c6 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2157,7 +2157,6 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn char* pStart = pRetrieveRsp->data; while(index++ < pRetrieveRsp->numOfBlocks) { SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); - blockDataEnsureCapacity(pb, pRetrieveRsp->numOfRows); code = extractDataBlockFromFetchRsp(pb, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 16f35b1b0d..4dd5e4ec15 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -658,7 +658,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataDestroy(p); - qDebug("%s get sorted row blocks, rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.rows); + qDebug("%s get sorted block, groupId:%0x"PRIx64" rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } From 99cabaee18cf41178b38108aa9c97d3bd1ec18dd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 14:07:51 +0800 Subject: [PATCH 62/96] fix(query):set correct offset. --- source/client/src/tmq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 8f866bde8e..24997a735c 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -2947,7 +2947,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; - char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols) - numOfCols * (sizeof(int8_t) + sizeof(int32_t)); + char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); int32_t* colLength = (int32_t*)pStart; pStart += sizeof(int32_t) * numOfCols; @@ -2977,6 +2977,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) char* data = pCol[k].pData + pCol[k].offset[j]; tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); } } else { From 249e2b12f66097d88882d8bcd19ccba6a9785216 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 10 Aug 2022 14:21:31 +0800 Subject: [PATCH 63/96] fix(query): fix interp + fill interpolation after last datablock --- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 +++ source/libs/executor/src/timewindowoperator.c | 35 ++++++++++--------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index cadb6fefd2..dd14aa6dc4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2631,6 +2631,11 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl STsdbReader* pPrevReader = pReader->innerReader[0]; SDataBlockIter* pBlockIter = &pPrevReader->status.blockIter; + code = tsdbTakeReadSnap(pPrevReader->pTsdb, &pPrevReader->pReadSnap); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + initFilesetIterator(&pPrevReader->status.fileIter, pPrevReader->pReadSnap->fs.aDFileSet, pPrevReader->order, pPrevReader->idStr); resetDataBlockIterator(&pPrevReader->status.blockIter, pPrevReader->order, pReader->status.pTableMap); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 51fea5ac24..0c62872299 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2115,7 +2115,7 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } -static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, +static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; @@ -2124,10 +2124,10 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp // output the result for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + //SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); switch (pSliceInfo->fillType) { @@ -2383,7 +2383,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2422,7 +2422,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2443,7 +2443,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2467,7 +2467,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepNextRows(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2501,7 +2501,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2532,15 +2532,16 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } - // check if need to interpolate after ts range - // except for fill(next), fill(linear) - while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (pResBlock->info.rows >= pResBlock->info.capacity) { - break; - } + } + + // check if need to interpolate after last datablock + // except for fill(next), fill(linear) + while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; } } From 465e79258afc01e8ed0dd52355cd2d564077c5d0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 10 Aug 2022 14:33:23 +0800 Subject: [PATCH 64/96] refact: a little optimize --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 73 ++++++++++++---------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 49ff8a732f..8ae0e824cf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -393,15 +393,16 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p SMemSkipListNode *px; SMemSkipListNode *pn; TSDBKEY *pTKey; - int c; - int backward = flags & SL_MOVE_BACKWARD; - int fromPos = flags & SL_MOVE_FROM_POS; + int32_t backward = flags & SL_MOVE_BACKWARD; + int32_t fromPos = flags & SL_MOVE_FROM_POS; if (backward) { px = pTbData->sl.pTail; - for (int8_t iLevel = pTbData->sl.maxLevel - 1; iLevel >= pTbData->sl.level; iLevel--) { - pos[iLevel] = px; + if (!fromPos) { + for (int8_t iLevel = pTbData->sl.level; iLevel < pTbData->sl.maxLevel; iLevel++) { + pos[iLevel] = px; + } } if (pTbData->sl.level) { @@ -412,7 +413,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p while (pn != pTbData->sl.pHead) { pTKey = (TSDBKEY *)SL_NODE_DATA(pn); - c = tsdbKeyCmprFn(pTKey, pKey); + int32_t c = tsdbKeyCmprFn(pTKey, pKey); if (c <= 0) { break; } else { @@ -427,8 +428,10 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p } else { px = pTbData->sl.pHead; - for (int8_t iLevel = pTbData->sl.maxLevel - 1; iLevel >= pTbData->sl.level; iLevel--) { - pos[iLevel] = px; + if (!fromPos) { + for (int8_t iLevel = pTbData->sl.level; iLevel < pTbData->sl.maxLevel; iLevel++) { + pos[iLevel] = px; + } } if (pTbData->sl.level) { @@ -437,9 +440,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { pn = SL_NODE_FORWARD(px, iLevel); while (pn != pTbData->sl.pTail) { - pTKey = (TSDBKEY *)SL_NODE_DATA(pn); - - c = tsdbKeyCmprFn(pTKey, pKey); + int32_t c = tsdbKeyCmprFn(SL_NODE_DATA(pn), pKey); if (c >= 0) { break; } else { @@ -480,36 +481,44 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN goto _exit; } pNode->level = level; - for (int8_t iLevel = 0; iLevel < level; iLevel++) { - SL_NODE_FORWARD(pNode, iLevel) = NULL; - SL_NODE_BACKWARD(pNode, iLevel) = NULL; - } - tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), pRow); - // put - for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) { - SMemSkipListNode *px = pos[iLevel]; + for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { + SMemSkipListNode *pn = pos[iLevel]; + SMemSkipListNode *px; if (forward) { - SMemSkipListNode *pNext = SL_NODE_FORWARD(px, iLevel); - - SL_NODE_FORWARD(pNode, iLevel) = pNext; - SL_NODE_BACKWARD(pNode, iLevel) = px; - - SL_NODE_BACKWARD(pNext, iLevel) = pNode; - SL_NODE_FORWARD(px, iLevel) = pNode; - } else { - SMemSkipListNode *pPrev = SL_NODE_BACKWARD(px, iLevel); + px = SL_NODE_FORWARD(pn, iLevel); + SL_NODE_BACKWARD(pNode, iLevel) = pn; SL_NODE_FORWARD(pNode, iLevel) = px; - SL_NODE_BACKWARD(pNode, iLevel) = pPrev; + } else { + px = SL_NODE_BACKWARD(pn, iLevel); - SL_NODE_FORWARD(pPrev, iLevel) = pNode; - SL_NODE_BACKWARD(px, iLevel) = pNode; + SL_NODE_BACKWARD(pNode, iLevel) = px; + SL_NODE_FORWARD(pNode, iLevel) = pn; } } + for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { + SMemSkipListNode *pn = pos[iLevel]; + SMemSkipListNode *px; + + if (forward) { + px = SL_NODE_FORWARD(pn, iLevel); + + SL_NODE_FORWARD(pn, iLevel) = pNode; + SL_NODE_BACKWARD(px, iLevel) = pNode; + } else { + px = SL_NODE_BACKWARD(pn, iLevel); + + SL_NODE_FORWARD(px, iLevel) = pNode; + SL_NODE_BACKWARD(pn, iLevel) = pNode; + } + + pos[iLevel] = pNode; + } + pTbData->sl.size++; if (pTbData->sl.level < pNode->level) { pTbData->sl.level = pNode->level; @@ -548,7 +557,7 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i // forward put rest data row.pTSRow = tGetSubmitBlkNext(&blkIter); if (row.pTSRow) { - for (int8_t iLevel = 0; iLevel < pTbData->sl.maxLevel; iLevel++) { + for (int8_t iLevel = pos[0]->level; iLevel < pTbData->sl.maxLevel; iLevel++) { pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); } do { From 6d267810a2471a9edb22d3d51b3cba720353d5d0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 15:22:40 +0800 Subject: [PATCH 65/96] fix(query): fix the invalid length for json data. --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5c37e6dc15..acdb3b68b0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1747,7 +1747,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int pStart1 += colLen1; } - *(int32_t*)(pResultInfo->convertJson) = totalLen; + *(int32_t*)(pResultInfo->convertJson + 4) = totalLen; pResultInfo->pData = pResultInfo->convertJson; return TSDB_CODE_SUCCESS; } From 6dfa2836711a6a2a7b140955a79654c9402c11b9 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 10 Aug 2022 15:35:00 +0800 Subject: [PATCH 66/96] fix: use the pre agg value as double when generate block sma for float column and double column --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d612e9bb10..60f0b18a62 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1520,22 +1520,22 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { break; } case TSDB_DATA_TYPE_FLOAT: { - pColAgg->sum += colVal.value.f; - if (pColAgg->min > colVal.value.f) { - pColAgg->min = colVal.value.f; + *(double*)(&pColAgg->sum) += colVal.value.f; + if (*(double*)(&pColAgg->min) > colVal.value.f) { + *(double*)(&pColAgg->min) = colVal.value.f; } - if (pColAgg->max < colVal.value.f) { - pColAgg->max = colVal.value.f; + if (*(double*)(&pColAgg->max) < colVal.value.f) { + *(double*)(&pColAgg->max) = colVal.value.f; } break; } case TSDB_DATA_TYPE_DOUBLE: { - pColAgg->sum += colVal.value.d; - if (pColAgg->min > colVal.value.d) { - pColAgg->min = colVal.value.d; + *(double*)(&pColAgg->sum) += colVal.value.d; + if (*(double*)(&pColAgg->min) > colVal.value.d) { + *(double*)(&pColAgg->min) = colVal.value.d; } - if (pColAgg->max < colVal.value.d) { - pColAgg->max = colVal.value.d; + if (*(double*)(&pColAgg->max) < colVal.value.d) { + *(double*)(&pColAgg->max) = colVal.value.d; } break; } From 399d240df593b1b50ea7ad813823997c9f1d83fa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 10 Aug 2022 15:41:12 +0800 Subject: [PATCH 67/96] refactor index --- source/libs/index/src/index.c | 1 - source/libs/index/src/indexCache.c | 3 +-- source/libs/index/src/indexUtil.c | 3 ++- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index d61cd871ee..850ddd4970 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -485,7 +485,6 @@ static void idxMayMergeTempToFinalRslt(SArray* result, TFileValue* tfv, SIdxTRsl // handle last iterator idxTRsltMergeTo(tr, lv->tableId); } else { - // temp result saved in help tfileValueDestroy(tfv); } } else { diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 50745cd3fb..794b85d244 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -45,7 +45,7 @@ static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* ct, SIdxTRslt* tr, static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); static int32_t cacheSearchRange(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); -/*comm func of compare, used in (LE/LT/GE/GT compare)*/ +/*comm compare func, used in (LE/LT/GE/GT compare)*/ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s, RangeType type); static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); static int32_t cacheSearchEqual_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); @@ -94,7 +94,6 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe if (0 == strcmp(c->colVal, pCt->colVal) && strlen(pCt->colVal) == strlen(c->colVal)) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid) - // taosArrayPush(result, &c->uid); *s = kTypeValue; } else if (c->operaType == DEL_VALUE) { INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid) diff --git a/source/libs/index/src/indexUtil.c b/source/libs/index/src/indexUtil.c index 1d20278895..3d083c1817 100644 --- a/source/libs/index/src/indexUtil.c +++ b/source/libs/index/src/indexUtil.c @@ -152,7 +152,7 @@ int verdataCompare(const void *a, const void *b) { int32_t cmp = compareUint64Val(&va->data, &vb->data); if (cmp == 0) { - cmp = 0 - compareUint32Val(&va->ver, &vb->data); + cmp = 0 - compareUint32Val(&va->ver, &vb->ver); return cmp; } return cmp; @@ -181,6 +181,7 @@ void idxTRsltDestroy(SIdxTRslt *tr) { taosArrayDestroy(tr->total); taosArrayDestroy(tr->add); taosArrayDestroy(tr->del); + taosMemoryFree(tr); } void idxTRsltMergeTo(SIdxTRslt *tr, SArray *result) { taosArraySort(tr->total, uidCompare); From 3327ae1556c14e02c1c2cb10672b09a6a3ad5b6f Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Wed, 10 Aug 2022 15:45:14 +0800 Subject: [PATCH 68/96] fix(taosAdapter): adapt to new raw_block --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 7a12413377..ee1e7d3b31 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 766dcc4 + GIT_TAG ed6a160 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 017a6c76662623f87231dad9436c7b29f1240a14 Mon Sep 17 00:00:00 2001 From: Bo Xiao Date: Wed, 10 Aug 2022 15:28:26 +0800 Subject: [PATCH 69/96] Update cpp.mdx --- docs/zh/14-reference/03-connector/cpp.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-connector/cpp.mdx b/docs/zh/14-reference/03-connector/cpp.mdx index aecf9fde12..3a8367ef33 100644 --- a/docs/zh/14-reference/03-connector/cpp.mdx +++ b/docs/zh/14-reference/03-connector/cpp.mdx @@ -279,7 +279,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 2. 调用 `taos_stmt_prepare()` 解析 INSERT 语句； 3. 如果 INSERT 语句中预留了表名但没有预留 TAGS，那么调用 `taos_stmt_set_tbname()` 来设置表名； 4. 如果 INSERT 语句中既预留了表名又预留了 TAGS（例如 INSERT 语句采取的是自动建表的方式），那么调用 `taos_stmt_set_tbname_tags()` 来设置表名和 TAGS 的值； -5. 调用 `taos_stmt_bind_param_batch()` 以多列的方式设置 VALUES 的值，或者调用 `taos_stmt_bind_param()` 以单行的方式设置 VALUES 的值； +5. 调用 `taos_stmt_bind_param_batch()` 以多行的方式设置 VALUES 的值，或者调用 `taos_stmt_bind_param()` 以单行的方式设置 VALUES 的值； 6. 调用 `taos_stmt_add_batch()` 把当前绑定的参数加入批处理； 7. 可以重复第 3 ～ 6 步，为批处理加入更多的数据行； 8. 调用 `taos_stmt_execute()` 执行已经准备好的批处理指令； From 18f6fb6123d8fb9386b821d04bd961cd252210bd Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 10 Aug 2022 16:04:03 +0800 Subject: [PATCH 70/96] enh(tmq): speed up consumer recover --- examples/c/tmq.c | 66 ++-- include/libs/stream/tstream.h | 2 +- include/util/taoserror.h | 1 + source/client/src/tmq.c | 173 +++-------- source/common/src/systable.c | 7 +- source/dnode/mnode/impl/src/mndConsumer.c | 12 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tq/tq.c | 3 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/stream/inc/streamInc.h | 2 +- source/libs/stream/src/streamData.c | 6 +- source/libs/stream/src/streamDispatch.c | 76 +---- source/libs/stream/src/streamExec.c | 34 +- source/libs/wal/src/walWrite.c | 2 + source/os/src/osSemaphore.c | 358 ++++++++++------------ source/util/src/terror.c | 1 + 16 files changed, 281 insertions(+), 466 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 1cdd4c02da..fc34915fe7 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -21,17 +21,17 @@ #include "taos.h" static int running = 1; -static char dbName[64] = "tmqdb"; -static char stbName[64] = "stb"; +static char dbName[64] = "tmqdb"; +static char stbName[64] = "stb"; static char topicName[64] = "topicname"; static int32_t msg_process(TAOS_RES* msg) { - char buf[1024]; + char buf[1024]; int32_t rows = 0; const char* topicName = tmq_get_topic_name(msg); - const char* dbName = tmq_get_db_name(msg); - int32_t vgroupId = tmq_get_vgroup_id(msg); + const char* dbName = tmq_get_db_name(msg); + int32_t vgroupId = tmq_get_vgroup_id(msg); printf("topic: %s\n", topicName); printf("db: %s\n", dbName); @@ -41,14 +41,14 @@ static int32_t msg_process(TAOS_RES* msg) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; - TAOS_FIELD* fields = taos_fetch_fields(msg); + TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); - int32_t* length = taos_fetch_lengths(msg); - int32_t precision = taos_result_precision(msg); - const char* tbName = tmq_get_table_name(msg); - rows++; + int32_t* length = taos_fetch_lengths(msg); + int32_t precision = taos_result_precision(msg); + const char* tbName = tmq_get_table_name(msg); + rows++; taos_print_row(buf, row, fields, numOfFields); - printf("row content from %s: %s\n", (tbName != NULL ? tbName : "null table"), buf); + printf("row content from %s: %s\n", (tbName != NULL ? tbName : "table null"), buf); } return rows; @@ -80,7 +80,8 @@ static int32_t init_env() { // create super table printf("create super table\n"); - pRes = taos_query(pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))"); + pRes = taos_query( + pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))"); if (taos_errno(pRes) != 0) { printf("failed to create super table stb, reason:%s\n", taos_errstr(pRes)); return -1; @@ -166,7 +167,6 @@ int32_t create_topic() { } taos_free_result(pRes); - // pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); pRes = taos_query(pConn, "create topic topicname as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topicname, reason:%s\n", taos_errstr(pRes)); @@ -184,26 +184,28 @@ void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { tmq_t* build_consumer() { tmq_conf_res_t code; - tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_t* conf = tmq_conf_new(); code = tmq_conf_set(conf, "enable.auto.commit", "true"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "group.id", "cgrpName"); if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "client.id", "user defined name"); + if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "td.connect.user", "root"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "td.connect.pass", "taosdata"); if (TMQ_CONF_OK != code) return NULL; - code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); + code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "experimental.snapshot.enable", "true"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "msg.with.table.name", "true"); if (TMQ_CONF_OK != code) return NULL; - - tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); - + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); tmq_conf_destroy(conf); return tmq; @@ -211,7 +213,7 @@ tmq_t* build_consumer() { tmq_list_t* build_topic_list() { tmq_list_t* topicList = tmq_list_new(); - int32_t code = tmq_list_append(topicList, "topicname"); + int32_t code = tmq_list_append(topicList, "topicname"); if (code) { return NULL; } @@ -228,18 +230,18 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topicList) { int32_t totalRows = 0; int32_t msgCnt = 0; - int32_t consumeDelay = 5000; + int32_t timeout = 5000; while (running) { - TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, consumeDelay); + TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, timeout); if (tmqmsg) { msgCnt++; totalRows += msg_process(tmqmsg); taos_free_result(tmqmsg); - } else { - break; - } + /*} else {*/ + /*break;*/ + } } - + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); } @@ -256,32 +258,30 @@ int main(int argc, char* argv[]) { tmq_t* tmq = build_consumer(); if (NULL == tmq) { - fprintf(stderr, "%% build_consumer() fail!\n"); + fprintf(stderr, "%% build_consumer() fail!\n"); return -1; } tmq_list_t* topic_list = build_topic_list(); if (NULL == topic_list) { return -1; - } - + } + basic_consume_loop(tmq, topic_list); code = tmq_unsubscribe(tmq); if (code) { fprintf(stderr, "%% Failed to unsubscribe: %s\n", tmq_err2str(code)); - } - else { + } else { fprintf(stderr, "%% unsubscribe\n"); } code = tmq_consumer_close(tmq); if (code) { fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); - } - else { + } else { fprintf(stderr, "%% Consumer closed\n"); } - + return 0; } diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9355d76dcb..32dc9e1866 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -118,7 +118,7 @@ typedef struct { int64_t sourceVer; int64_t reqId; - SArray* blocks; // SArray + SArray* blocks; // SArray } SStreamDataBlock; typedef struct { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 27fb057b44..145e2f7aad 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -617,6 +617,7 @@ int32_t* taosGetErrno(); //tmq #define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000) +#define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001) #ifdef __cplusplus } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 073f772ee4..c4375c85b4 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -122,6 +122,7 @@ enum { TMQ_CONSUMER_STATUS__INIT = 0, TMQ_CONSUMER_STATUS__READY, TMQ_CONSUMER_STATUS__NO_TOPIC, + TMQ_CONSUMER_STATUS__RECOVER, }; enum { @@ -134,10 +135,8 @@ typedef struct { // statistics int64_t pollCnt; // offset - /*int64_t committedOffset;*/ - /*int64_t currentOffset;*/ - STqOffsetVal committedOffsetNew; - STqOffsetVal currentOffsetNew; + STqOffsetVal committedOffset; + STqOffsetVal currentOffset; // connection info int32_t vgId; int32_t vgStatus; @@ -152,7 +151,6 @@ typedef struct { SArray* vgs; // SArray - int8_t isSchemaAdaptive; SSchemaWrapper schema; } SMqClientTopic; @@ -190,10 +188,9 @@ typedef struct { } SMqPollCbParam; typedef struct { - tmq_t* tmq; - int8_t automatic; - int8_t async; - /*int8_t freeOffsets;*/ + tmq_t* tmq; + int8_t automatic; + int8_t async; int32_t waitingRspNum; int32_t totalRspNum; int32_t rspErr; @@ -418,7 +415,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pOffset->val = pVg->currentOffsetNew; + pOffset->val = pVg->currentOffset; int32_t groupLen = strlen(tmq->groupId); memcpy(pOffset->subKey, tmq->groupId, groupLen); @@ -462,7 +459,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT pVg->vgId, pOffset->val.version); // TODO: put into cb - pVg->committedOffsetNew = pVg->currentOffsetNew; + pVg->committedOffset = pVg->currentOffset; pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; @@ -504,7 +501,6 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm pParamSet->tmq = tmq; pParamSet->automatic = 0; pParamSet->async = async; - /*pParamSet->freeOffsets = 1;*/ pParamSet->userCb = userCb; pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); @@ -518,7 +514,7 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); if (pVg->vgId != vgId) continue; - if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { goto FAIL; } @@ -550,8 +546,8 @@ FAIL: return 0; } -int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, - void* userParam) { +int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, + void* userParam) { int32_t code = -1; if (msg != NULL) { @@ -566,7 +562,6 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ pParamSet->tmq = tmq; pParamSet->automatic = automatic; pParamSet->async = async; - /*pParamSet->freeOffsets = 1;*/ pParamSet->userCb = userCb; pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); @@ -583,7 +578,9 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ tscDebug("consumer:%" PRId64 ", begin commit for topic %s, vgId:%d", tmq->consumerId, pTopic->topicName, pVg->vgId); - if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { + tscDebug("consumer: %ld, vg:%d, current %ld, committed %ld", tmq->consumerId, pVg->vgId, + pVg->currentOffset.version, pVg->committedOffset.version); if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { continue; } @@ -699,7 +696,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { tmqAskEp(tmq, true); taosTmrReset(tmqAssignAskEpTask, 1000, tmq, tmqMgmt.timer, &tmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { - tmqCommitInner2(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); + tmqCommitInner(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) { } else { @@ -888,12 +885,6 @@ FAIL: return NULL; } -#if 0 -int32_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { - return tmqCommitInner2(tmq, offsets, 0, async, tmq->commitCb, tmq->commitCbUserParam); -} -#endif - int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); @@ -967,7 +958,11 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = param.rspErr; if (code != 0) goto FAIL; + int32_t retryCnt = 0; while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + if (retryCnt++ > 10) { + goto FAIL; + } tscDebug("consumer not ready, retry"); taosMsleep(500); } @@ -1006,8 +1001,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t epoch = pParam->epoch; taosMemoryFree(pParam); if (code != 0) { - tscWarn("msg discard from vgId:%d, epoch %d, code:%x", vgId, epoch, code); + tscWarn("msg discard from vgId:%d, epoch %d, since %s", vgId, epoch, terrstr()); if (pMsg->pData) taosMemoryFree(pMsg->pData); + if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); + goto CREATE_MSG_FAIL; + } if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { @@ -1083,7 +1082,7 @@ CREATE_MSG_FAIL: return -1; } -bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { +bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); @@ -1112,10 +1111,10 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); char buf[80]; - tFormatOffset(buf, 80, &pVgCur->currentOffsetNew); + tFormatOffset(buf, 80, &pVgCur->currentOffset); tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); - taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffsetNew, sizeof(STqOffsetVal)); + taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(STqOffsetVal)); } } } @@ -1142,7 +1141,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { SMqClientVg clientVg = { .pollCnt = 0, - .currentOffsetNew = offsetNew, + .currentOffset = offsetNew, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, @@ -1166,93 +1165,6 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { return set; } -#if 0 -bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { - /*printf("call update ep %d\n", epoch);*/ - bool set = false; - int32_t topicNumGet = taosArrayGetSize(pRsp->topics); - char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; - tscDebug("consumer:%" PRId64 ", update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, - topicNumGet); - SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); - if (newTopics == NULL) { - return false; - } - SHashObj* pHash = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); - if (pHash == NULL) { - taosArrayDestroy(newTopics); - return false; - } - - // find topic, build hash - for (int32_t i = 0; i < topicNumGet; i++) { - SMqClientTopic topic = {0}; - SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); - topic.schema = pTopicEp->schema; - taosHashClear(pHash); - topic.topicName = strdup(pTopicEp->topic); - tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN); - - tscDebug("consumer:%" PRId64 ", update topic: %s", tmq->consumerId, topic.topicName); - int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); - for (int32_t j = 0; j < topicNumCur; j++) { - // find old topic - SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); - if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) { - int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); - tscDebug("consumer:%" PRId64 ", new vg num: %d", tmq->consumerId, vgNumCur); - if (vgNumCur == 0) break; - for (int32_t k = 0; k < vgNumCur; k++) { - SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); - sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); - tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey); - taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); - } - break; - } - } - - int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); - topic.vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); - for (int32_t j = 0; j < vgNumGet; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); - int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); - int64_t offset = pVgEp->offset; - tscDebug("consumer:%" PRId64 ", (epoch %d) original offset of vgId:%d is %" PRId64, tmq->consumerId, epoch, pVgEp->vgId, offset); - if (pOffset != NULL) { - offset = *pOffset; - tscDebug("consumer:%" PRId64 ", (epoch %d) receive offset of vgId:%d, full key is %s", tmq->consumerId, epoch, pVgEp->vgId, - vgKey); - } - tscDebug("consumer:%" PRId64 ", (epoch %d) offset of vgId:%d updated to %" PRId64, tmq->consumerId, epoch, pVgEp->vgId, offset); - SMqClientVg clientVg = { - .pollCnt = 0, - .currentOffset = offset, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet, - .vgStatus = TMQ_VG_STATUS__IDLE, - .vgSkipCnt = 0, - }; - taosArrayPush(topic.vgs, &clientVg); - set = true; - } - taosArrayPush(newTopics, &topic); - } - if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); - taosHashCleanup(pHash); - tmq->clientTopics = newTopics; - - if (taosArrayGetSize(tmq->clientTopics) == 0) - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC); - else - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); - - atomic_store_32(&tmq->epoch, epoch); - return set; -} -#endif - int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; @@ -1278,7 +1190,7 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); /*printf("rsp epoch %" PRId64 " sz %" PRId64 "\n", rsp.epoch, rsp.topics->size);*/ /*printf("tmq epoch %" PRId64 " sz %" PRId64 "\n", tmq->epoch, tmq->clientTopics->size);*/ - tmqUpdateEp2(tmq, head->epoch, &rsp); + tmqUpdateEp(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM); @@ -1430,7 +1342,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; /*pReq->currentOffset = reqOffset;*/ - pReq->reqOffset = pVg->currentOffsetNew; + pReq->reqOffset = pVg->currentOffset; pReq->reqId = generateRequestId(); pReq->useSnapshot = tmq->useSnapshot; @@ -1534,7 +1446,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { /*printf("send poll\n");*/ char offsetFormatBuf[80]; - tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffsetNew); + tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffset); tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64, tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); /*printf("send vgId:%d %" PRId64 "\n", pVg->vgId, pVg->currentOffset);*/ @@ -1552,7 +1464,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) { SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; - tmqUpdateEp2(tmq, rspWrapper->epoch, rspMsg); + tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg); /*tmqClearUnhandleMsg(tmq);*/ *pReset = true; } else { @@ -1586,7 +1498,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, * rspMsg->msg.rspOffset);*/ - pVg->currentOffsetNew = pollRspWrapper->dataRsp.rspOffset; + pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); if (pollRspWrapper->dataRsp.blockNum == 0) { taosFreeQitem(pollRspWrapper); @@ -1609,8 +1521,8 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, * rspMsg->msg.rspOffset);*/ - pVg->currentOffsetNew.version = pollRspWrapper->metaRsp.rspOffset; - pVg->currentOffsetNew.type = TMQ_OFFSET__LOG; + pVg->currentOffset.version = pollRspWrapper->metaRsp.rspOffset; + pVg->currentOffset.type = TMQ_OFFSET__LOG; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); @@ -1653,6 +1565,17 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { return NULL; } + if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { + int32_t retryCnt = 0; + while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + if (retryCnt++ > 10) { + return NULL; + } + tscDebug("consumer not ready, retry"); + taosMsleep(500); + } + } + while (1) { tmqHandleAllDelayedTask(tmq); if (tmqPollImpl(tmq, timeout) < 0) return NULL; @@ -3381,10 +3304,10 @@ int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { // - tmqCommitInner2(tmq, msg, 0, 1, cb, param); + tmqCommitInner(tmq, msg, 0, 1, cb, param); } int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) { // - return tmqCommitInner2(tmq, msg, 0, 0, NULL, NULL); + return tmqCommitInner(tmq, msg, 0, 0, NULL, NULL); } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 675d3bbfdd..edd718090f 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -16,8 +16,8 @@ #include "systable.h" #include "taos.h" #include "tdef.h" -#include "types.h" #include "tgrant.h" +#include "types.h" #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) @@ -243,8 +243,8 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, -// {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, -// {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, + // {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, + // {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema)}, {TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema)}, {TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, @@ -284,7 +284,6 @@ static const SSysDbTableSchema consumerSchema[] = { {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "topics", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 39b57f2966..3cdc53cd2c 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -131,8 +131,9 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId, mndConsumerStatusName(pConsumer->status)); - if (pConsumer->status != MQ_CONSUMER_STATUS__READY) { + if (pConsumer->status != MQ_CONSUMER_STATUS__LOST_REBD) { mndReleaseConsumer(pMnode, pConsumer); + terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } @@ -275,6 +276,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int32_t status = atomic_load_32(&pConsumer->status); if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + mInfo("try to recover consumer %ld", consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -305,15 +307,14 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); -#if 1 atomic_store_32(&pConsumer->hbStatus, 0); -#endif // 1. check consumer status int32_t status = atomic_load_32(&pConsumer->status); #if 1 if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + mInfo("try to recover consumer %ld", consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -326,6 +327,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { #endif if (status != MQ_CONSUMER_STATUS__READY) { + mInfo("consumer %ld not ready, status: %s", consumerId, mndConsumerStatusName(status)); terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } @@ -939,10 +941,6 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * colDataAppend(pColInfo, numOfRows, NULL, true); } - // pid - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->pid, true); - // end point pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 4c6320ecb5..bce2b642e8 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -157,7 +157,7 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); int32_t tqProcessCheckAlterInfoReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); -int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen, int64_t ver); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 02ca248054..112543e340 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -183,7 +183,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con return 0; } -int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen, int64_t ver) { STqOffset offset = {0}; SDecoder decoder; tDecoderInit(&decoder, msg, msgLen); @@ -302,6 +302,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqError("tmq poll: consumer handle mismatch for consumer:%" PRId64 ", in vgId:%d, subkey %s, handle consumer id %" PRId64, consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0cb66484c0..2d181c5458 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -204,7 +204,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp break; case TDMT_VND_MQ_COMMIT_OFFSET: if (tqProcessOffsetCommitReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), - pMsg->contLen - sizeof(SMsgHead)) < 0) { + pMsg->contLen - sizeof(SMsgHead), version) < 0) { goto _err; } break; diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 7dc8d822e9..3776cb261f 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -44,7 +44,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); -SStreamQueueItem* streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); void streamFreeQitem(SStreamQueueItem* data); #ifdef __cplusplus diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 54014d7df9..c584c2a15f 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -125,7 +125,7 @@ void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) { } } -SStreamQueueItem* streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { ASSERT(elem); if (dst->type == STREAM_INPUT__DATA_BLOCK && elem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; @@ -172,8 +172,8 @@ void streamFreeQitem(SStreamQueueItem* data) { int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { - void* data = taosArrayGetP(pMerge->reqs, i); - taosMemoryFree(data); + void* dataStr = taosArrayGetP(pMerge->reqs, i); + taosMemoryFree(dataStr); taosMemoryFree(pRef); } } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9d98afa65a..c78ff0756f 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -370,80 +370,6 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat return 0; } -int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { - void* buf = NULL; - int32_t code = -1; - int32_t blockNum = taosArrayGetSize(data->blocks); - ASSERT(blockNum != 0); - - SStreamDispatchReq req = { - .streamId = pTask->streamId, - .dataSrcVgId = data->srcVgId, - .upstreamTaskId = pTask->taskId, - .upstreamChildId = pTask->selfChildId, - .upstreamNodeId = pTask->nodeId, - .blockNum = blockNum, - }; - - req.data = taosArrayInit(blockNum, sizeof(void*)); - req.dataLen = taosArrayInit(blockNum, sizeof(int32_t)); - if (req.data == NULL || req.dataLen == NULL) { - goto FAIL; - } - for (int32_t i = 0; i < blockNum; i++) { - SSDataBlock* pDataBlock = taosArrayGet(data->blocks, i); - if (streamAddBlockToDispatchMsg(pDataBlock, &req) < 0) { - goto FAIL; - } - } - int32_t vgId = 0; - int32_t downstreamTaskId = 0; - // find ep - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - vgId = pTask->fixedEpDispatcher.nodeId; - *ppEpSet = &pTask->fixedEpDispatcher.epSet; - downstreamTaskId = pTask->fixedEpDispatcher.taskId; - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - // TODO get ctbName for each block - SSDataBlock* pBlock = taosArrayGet(data->blocks, 0); - char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId); - // TODO: get hash function by hashMethod - - // get groupId, compute hash value - uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); - - // get node - // TODO: optimize search process - SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t sz = taosArrayGetSize(vgInfo); - for (int32_t i = 0; i < sz; i++) { - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - ASSERT(pVgInfo->vgId > 0); - if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { - vgId = pVgInfo->vgId; - downstreamTaskId = pVgInfo->taskId; - *ppEpSet = &pVgInfo->epSet; - break; - } - } - } - - ASSERT(vgId > 0 || vgId == SNODE_HANDLE); - req.taskId = downstreamTaskId; - - qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, - downstreamTaskId, vgId); - - streamDispatchOneReq(pTask, &req, vgId, *ppEpSet); - - code = 0; -FAIL: - if (code < 0 && buf) rpcFreeCont(buf); - if (req.data) taosArrayDestroyP(req.data, (FDelete)taosMemoryFree); - if (req.dataLen) taosArrayDestroy(req.dataLen); - return code; -} - int32_t streamDispatch(SStreamTask* pTask) { ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); @@ -461,7 +387,7 @@ int32_t streamDispatch(SStreamTask* pTask) { } ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - qDebug("stream continue dispatching: task %d", pTask->taskId); + qDebug("stream dispatching: task %d", pTask->taskId); int32_t code = 0; if (streamDispatchAllBlocks(pTask, pBlock) < 0) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 72249c5181..06ca26f029 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -82,17 +82,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* return 0; } -#if 0 -static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock* pBlock) { - ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - int32_t childId = pBlock->childId; - int64_t ver = pBlock->sourceVer; - SStreamChildEpInfo* pChildInfo = taosArrayGetP(pTask->childEpInfo, childId); - /*pChildInfo-> = ver;*/ - return 0; -} -#endif - int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch) { ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); @@ -150,10 +139,11 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch) return 0; } + // TODO: handle version int32_t streamExecForAll(SStreamTask* pTask) { while (1) { - int32_t cnt = 1; + int32_t batchCnt = 1; void* data = NULL; while (1) { SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); @@ -169,13 +159,12 @@ int32_t streamExecForAll(SStreamTask* pTask) { } } else { void* newRet; - if ((newRet = streamAppendQueueItem(data, qItem)) == NULL) { + if ((newRet = streamMergeQueueItem(data, qItem)) == NULL) { streamQueueProcessFail(pTask->inputQueue); break; } else { - cnt++; + batchCnt++; data = newRet; - /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ streamQueueProcessSuccess(pTask->inputQueue); } } @@ -198,16 +187,14 @@ int32_t streamExecForAll(SStreamTask* pTask) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); + qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, batchCnt); streamTaskExecImpl(pTask, data, pRes); qDebug("stream task %d exec end", pTask->taskId); if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); if (qRes == NULL) { - // TODO log failed ver - streamQueueProcessFail(pTask->inputQueue); - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); streamFreeQitem(data); return -1; } @@ -218,17 +205,18 @@ int32_t streamExecForAll(SStreamTask* pTask) { SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; + } else if (((SStreamQueueItem*)data)->type == STREAM_INPUT__MERGED_SUBMIT) { + SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)data; + qRes->childId = pTask->selfChildId; + qRes->sourceVer = pMerged->ver; } if (streamTaskOutput(pTask, qRes) < 0) { - // TODO save failed ver - /*streamQueueProcessFail(pTask->inputQueue);*/ taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - taosFreeQitem(qRes); streamFreeQitem(data); + taosFreeQitem(qRes); return -1; } - /*streamQueueProcessSuccess(pTask->inputQueue);*/ } else { taosArrayDestroy(pRes); } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index ad571a9e82..fad3977a21 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -421,6 +421,8 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); + wDebug("vgId:%d, wal write log %ld, msgType: %s", pWal->cfg.vgId, index, TMSG_INFO(msgType)); + if (taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 63c414386b..ab4fb551fe 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -76,8 +76,8 @@ int32_t tsem_wait(tsem_t* sem) { int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { struct timespec ts, rel; - FILETIME ft_before, ft_after; - int rc; + FILETIME ft_before, ft_after; + int rc; rel.tv_sec = 0; rel.tv_nsec = nanosecs; @@ -217,7 +217,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int e = errno; // if (e == EEXIST) continue; // if (e == EINTR) continue; -// fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, +// fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem, // e, strerror(e)); // abort(); // } while (p->sem == SEM_FAILED); @@ -231,7 +232,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value); // if (ret != KERN_SUCCESS) { -// fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // // we fail-fast here, because we have less-doc about semaphore_create for the moment // abort(); @@ -258,8 +260,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // struct tsem_s *p = *sem; // if (!p->valid) { -// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); -// abort(); +// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); abort(); // } // #ifdef SEM_USE_PTHREAD // if (taosThreadMutexLock(&p->lock)) { @@ -270,7 +272,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // p->val -= 1; // if (p->val < 0) { // if (taosThreadCondWait(&p->cond, &p->lock)) { -// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // abort(); // } @@ -297,8 +300,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // struct tsem_s *p = *sem; // if (!p->valid) { -// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); -// abort(); +// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); abort(); // } // #ifdef SEM_USE_PTHREAD // if (taosThreadMutexLock(&p->lock)) { @@ -309,7 +312,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // p->val += 1; // if (p->val <= 0) { // if (taosThreadCondSignal(&p->cond)) { -// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // abort(); // } @@ -332,7 +336,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int tsem_destroy(tsem_t *sem) { // // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); // if (!*sem) { -// // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); +// // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); // // abort(); // return 0; // } @@ -370,7 +375,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int r = sem_unlink(name); // if (r) { // int e = errno; -// fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, +// fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem, // e, strerror(e)); // abort(); // } @@ -385,225 +391,189 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // *sem = NULL; // return 0; // } -typedef struct -{ - pthread_mutex_t count_lock; - pthread_cond_t count_bump; - unsigned int count; -}bosal_sem_t; +typedef struct { + pthread_mutex_t count_lock; + pthread_cond_t count_bump; + unsigned int count; +} bosal_sem_t; -int tsem_init(tsem_t *psem, int flags, unsigned int count) -{ - bosal_sem_t *pnewsem; - int result; +int tsem_init(tsem_t *psem, int flags, unsigned int count) { + bosal_sem_t *pnewsem; + int result; - pnewsem = (bosal_sem_t *)malloc(sizeof(bosal_sem_t)); - if (! pnewsem) - { - return -1; - } - result = pthread_mutex_init(&pnewsem->count_lock, NULL); - if (result) - { - free(pnewsem); - return result; - } - result = pthread_cond_init(&pnewsem->count_bump, NULL); - if (result) - { - pthread_mutex_destroy(&pnewsem->count_lock); - free(pnewsem); - return result; - } - pnewsem->count = count; - *psem = (tsem_t)pnewsem; - return 0; + pnewsem = (bosal_sem_t *)malloc(sizeof(bosal_sem_t)); + if (!pnewsem) { + return -1; + } + result = pthread_mutex_init(&pnewsem->count_lock, NULL); + if (result) { + free(pnewsem); + return result; + } + result = pthread_cond_init(&pnewsem->count_bump, NULL); + if (result) { + pthread_mutex_destroy(&pnewsem->count_lock); + free(pnewsem); + return result; + } + pnewsem->count = count; + *psem = (tsem_t)pnewsem; + return 0; } -int tsem_destroy(tsem_t *psem) -{ - bosal_sem_t *poldsem; +int tsem_destroy(tsem_t *psem) { + bosal_sem_t *poldsem; - if (! psem) - { - return EINVAL; - } - poldsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + poldsem = (bosal_sem_t *)*psem; - pthread_mutex_destroy(&poldsem->count_lock); - pthread_cond_destroy(&poldsem->count_bump); - free(poldsem); - return 0; + pthread_mutex_destroy(&poldsem->count_lock); + pthread_cond_destroy(&poldsem->count_bump); + free(poldsem); + return 0; } -int tsem_post(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_post(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - pxsem->count = pxsem->count + 1; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + pxsem->count = pxsem->count + 1; - xresult = pthread_cond_signal(&pxsem->count_bump); + xresult = pthread_cond_signal(&pxsem->count_bump); - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_trywait(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_trywait(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count > 0) - { - pxsem->count--; - } - else - { - xresult = EAGAIN; - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + if (pxsem->count > 0) { + pxsem->count--; + } else { + xresult = EAGAIN; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_wait(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_wait(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count == 0) - { - xresult = pthread_cond_wait(&pxsem->count_bump, &pxsem->count_lock); + if (pxsem->count == 0) { + xresult = pthread_cond_wait(&pxsem->count_bump, &pxsem->count_lock); + } + if (!xresult) { + if (pxsem->count > 0) { + pxsem->count--; } - if (! xresult) - { - if (pxsem->count > 0) - { - pxsem->count--; - } - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_timewait(tsem_t *psem, int64_t nanosecs) -{ +int tsem_timewait(tsem_t *psem, int64_t nanosecs) { struct timespec abstim = { .tv_sec = 0, .tv_nsec = nanosecs, }; - bosal_sem_t *pxsem; - int result, xresult; + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count == 0) - { - xresult = pthread_cond_timedwait(&pxsem->count_bump, &pxsem->count_lock, &abstim); + if (pxsem->count == 0) { + xresult = pthread_cond_timedwait(&pxsem->count_bump, &pxsem->count_lock, &abstim); + } + if (!xresult) { + if (pxsem->count > 0) { + pxsem->count--; } - if (! xresult) - { - if (pxsem->count > 0) - { - pxsem->count--; - } - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -bool taosCheckPthreadValid(TdThread thread) { +bool taosCheckPthreadValid(TdThread thread) { int32_t ret = taosThreadKill(thread, 0); if (ret == ESRCH) return false; if (ret == EINVAL) return false; // alive return true; - } +} int64_t taosGetSelfPthreadId() { TdThread thread = taosThreadSelf(); @@ -650,7 +620,13 @@ int64_t taosGetSelfPthreadId() { int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; } void taosResetPthread(TdThread* thread) { *thread = 0; } bool taosComparePthread(TdThread first, TdThread second) { return first == second; } -int32_t taosGetPId() { return getpid(); } + +int32_t taosGetPId() { + static __thread int32_t pid = 0; + if (pid != 0) return pid; + pid = getpid(); + return pid; +} int32_t taosGetAppName(char* name, int32_t* len) { const char* self = "/proc/self/exe"; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4780d85a30..93b91739a3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -621,6 +621,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Invalid index file" //tmq TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") #ifdef TAOS_ERROR_C }; From c8ffd4f1610d6099e569d9c73511b2cd045708b4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 10 Aug 2022 16:12:39 +0800 Subject: [PATCH 71/96] docs: update helm doc for 3.0 (#15938) --- docs/zh/10-deployment/05-helm.md | 47 ++++++++++---------------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/docs/zh/10-deployment/05-helm.md b/docs/zh/10-deployment/05-helm.md index 0bbd986b4b..9a723ff62f 100644 --- a/docs/zh/10-deployment/05-helm.md +++ b/docs/zh/10-deployment/05-helm.md @@ -22,7 +22,7 @@ Helm 会使用 kubectl 和 kubeconfig 的配置来操作 Kubernetes，可以参 TDengine Chart 尚未发布到 Helm 仓库，当前可以从 GitHub 直接下载： ```bash -wget https://github.com/taosdata/TDengine-Operator/raw/main/helm/tdengine-0.3.0.tgz +wget https://github.com/taosdata/TDengine-Operator/raw/3.0/helm/tdengine-3.0.0.tgz ``` @@ -38,7 +38,7 @@ kubectl get storageclass 之后，使用 helm 命令安装： ```bash -helm install tdengine tdengine-0.3.0.tgz \ +helm install tdengine tdengine-3.0.0.tgz \ --set storage.className= ``` @@ -46,7 +46,7 @@ helm install tdengine tdengine-0.3.0.tgz \ 在 minikube 环境下，可以设置一个较小的容量避免超出磁盘可用空间： ```bash -helm install tdengine tdengine-0.3.0.tgz \ +helm install tdengine tdengine-3.0.0.tgz \ --set storage.className=standard \ --set storage.dataSize=2Gi \ --set storage.logSize=10Mi @@ -83,14 +83,14 @@ TDengine 支持 `values.yaml` 自定义。 通过 helm show values 可以获取 TDengine Chart 支持的全部 values 列表： ```bash -helm show values tdengine-0.3.0.tgz +helm show values tdengine-3.0.0.tgz ``` 你可以将结果保存为 values.yaml，之后可以修改其中的各项参数，如 replica 数量，存储类名称，容量大小，TDengine 配置等，然后使用如下命令安装 TDengine 集群： ```bash -helm install tdengine tdengine-0.3.0.tgz -f values.yaml +helm install tdengine tdengine-3.0.0.tgz -f values.yaml ``` @@ -107,37 +107,17 @@ image: prefix: tdengine/tdengine #pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - #tag: "2.4.0.5" +# tag: "3.0.0.0" service: # ClusterIP is the default service type, use NodeIP only if you know what you are doing. type: ClusterIP ports: # TCP range required - tcp: - [ - 6030, - 6031, - 6032, - 6033, - 6034, - 6035, - 6036, - 6037, - 6038, - 6039, - 6040, - 6041, - 6042, - 6043, - 6044, - 6045, - 6060, - ] - # UDP range 6030-6039 - udp: [6030, 6031, 6032, 6033, 6034, 6035, 6036, 6037, 6038, 6039] + tcp: [6030, 6041, 6042, 6043, 6044, 6046, 6047, 6048, 6049, 6060] + # UDP range + udp: [6044, 6045] -arbitrator: true # Set timezone here, not in taoscfg timezone: "Asia/Shanghai" @@ -182,11 +162,14 @@ clusterDomainSuffix: "" # # Btw, keep quotes "" around the value like below, even the value will be number or not. taoscfg: + # Starts as cluster or not, must be 0 or 1. + # 0: all pods will start as a seperate TDengine server + # 1: pods will start as TDengine server cluster. [default] + CLUSTER: "1" + # number of replications, for cluster only TAOS_REPLICA: "1" - # number of management nodes in the system - TAOS_NUM_OF_MNODES: "1" # number of days per DB file # TAOS_DAYS: "10" @@ -422,7 +405,7 @@ kubectl --namespace default exec $POD_NAME -- taos -s 'drop dnode " Date: Wed, 10 Aug 2022 16:20:44 +0800 Subject: [PATCH 72/96] fix: return init agg failure when create operators to avoid coredump --- source/libs/executor/src/executorimpl.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7e77799702..47d0ed3131 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3432,7 +3432,11 @@ int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInf return code; } - doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey); + code = doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + for (int32_t i = 0; i < numOfCols; ++i) { pSup->pCtx[i].pBuf = pAggSup->pResultBuf; } From 788735f7e693d508c262cf2bb827303764432b21 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 10 Aug 2022 16:25:00 +0800 Subject: [PATCH 73/96] fix: add qerror log when create disk based result buf failed --- source/libs/executor/src/executorimpl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 47d0ed3131..bd56bc2745 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3413,6 +3413,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n } int32_t code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir); if (code != TSDB_CODE_SUCCESS) { + qError("Create agg result buf failed since %s", tstrerror(code)); return code; } From 6fc38165fb7d009d04878eec58be29a2958fd64b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 10 Aug 2022 16:29:55 +0800 Subject: [PATCH 74/96] fix(query): fix linear interpolation issue between two blocks TD-18309 --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/inc/tfill.h | 1 - source/libs/executor/src/timewindowoperator.c | 68 ++++++++++++++----- 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8439cf700d..75d7c1c1ff 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -739,6 +739,7 @@ typedef struct STimeSliceOperatorInfo { SArray* pPrevRow; // SArray SArray* pNextRow; // SArray SArray* pLinearInfo; // SArray + bool fillLastPoint; bool isPrevRowSet; bool isNextRowSet; int32_t fillType; // fill type diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index c2de48d0eb..1e95f4a04f 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -37,7 +37,6 @@ typedef struct SFillLinearInfo { SPoint start; SPoint end; bool hasNull; - bool fillLastPoint; int16_t type; int32_t bytes; } SFillLinearInfo; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0c62872299..c7fa444cc6 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2087,8 +2087,10 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock pSliceInfo->isNextRowSet = true; } -static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { +static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex, + bool isLastRow) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + bool fillLastPoint = pSliceInfo->fillLastPoint; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); @@ -2096,16 +2098,22 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo // null data should not be kept since it can not be used to perform interpolation if (!colDataIsNull_s(pColInfoData, i)) { - int64_t startKey = *(int64_t*)colDataGetData(pTsCol, rowIndex); - int64_t endKey = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1); - pLinearInfo->start.key = startKey; - pLinearInfo->end.key = endKey; + if (isLastRow) { + pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + } else if (fillLastPoint) { + pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + } else { + pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1); - char* val; - val = colDataGetData(pColInfoData, rowIndex); - memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes); - val = colDataGetData(pColInfoData, rowIndex + 1); - memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes); + char* val; + val = colDataGetData(pColInfoData, rowIndex); + memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes); + val = colDataGetData(pColInfoData, rowIndex + 1); + memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes); + } pLinearInfo->hasNull = false; } else { @@ -2113,6 +2121,8 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } } + pSliceInfo->fillLastPoint = isLastRow ? true : false; + } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, @@ -2269,7 +2279,7 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB } pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo)); - if (pInfo->pNextRow == NULL) { + if (pInfo->pLinearInfo == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -2283,15 +2293,20 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes); linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes); linearInfo.hasNull = false; - linearInfo.fillLastPoint = false; linearInfo.type = pColInfo->info.type; linearInfo.bytes = pColInfo->info.bytes; taosArrayPush(pInfo->pLinearInfo, &linearInfo); } + pInfo->fillLastPoint = false; + return TSDB_CODE_SUCCESS; } +static bool needToFillLastPoint(STimeSliceOperatorInfo* pSliceInfo) { + return (pSliceInfo->fillLastPoint == true && pSliceInfo->fillType == TSDB_FILL_LINEAR); +} + static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { int32_t code; code = initPrevRowsKeeper(pInfo, pBlock); @@ -2356,6 +2371,23 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { for (int32_t i = 0; i < pBlock->info.rows; ++i) { int64_t ts = *(int64_t*)colDataGetData(pTsCol, i); + if (i == 0 && needToFillLastPoint(pSliceInfo)) { // first row in current block + doKeepLinearInfo(pSliceInfo, pBlock, i, false); + while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } + + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; + } + } + if (ts == pSliceInfo->current) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; @@ -2374,9 +2406,10 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // for linear interpolation, always fill value between this and next points; // if its the first point in data block, also fill values between previous(if there's any) and this point; - // if its the last point in data block, no need to fill, but reserve this point as the start value for next data block. + // if its the last point in data block, no need to fill, but reserve this point as the start value and do + // the interpolation when processing next data block. if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - doKeepLinearInfo(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (i < pBlock->info.rows - 1) { @@ -2396,6 +2429,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } + } else {// it is the last row of current block + //store ts value as start, and calculate interp value when processing next block + doKeepLinearInfo(pSliceInfo, pBlock, i, true); } } else { // non-linear interpolation pSliceInfo->current = @@ -2414,7 +2450,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepPrevRows(pSliceInfo, pBlock, i); if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - doKeepLinearInfo(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i, false); // no need to increate pSliceInfo->current here //pSliceInfo->current = // taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -2494,7 +2530,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - doKeepLinearInfo(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (i < pBlock->info.rows - 1) { From 5f7b7a55df5d3511c9bc563506a689fad79b52a6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 10 Aug 2022 16:33:12 +0800 Subject: [PATCH 75/96] fix: fix memory leaks --- source/libs/executor/src/joinoperator.c | 2 ++ source/libs/executor/src/scanoperator.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 2f9eff50b6..7d2b84d0f0 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -132,6 +132,8 @@ void destroyMergeJoinOperator(void* param, int32_t numOfOutput) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; nodesDestroyNode(pJoinOperator->pCondAfterMerge); + pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); + taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b21e937eb9..8e9ae3cd15 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2570,6 +2570,8 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { STagScanInfo* pInfo = (STagScanInfo*)param; pInfo->pRes = blockDataDestroy(pInfo->pRes); + taosArrayDestroy(pInfo->pColMatchInfo); + taosMemoryFreeClear(param); } From b4c8685766c1a8ca6d50d6fa63da2cb3e13ba5e4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 10 Aug 2022 17:17:30 +0800 Subject: [PATCH 76/96] refactor(sync): add leader, follower call back --- include/libs/sync/sync.h | 3 +++ source/dnode/mnode/impl/src/mndSync.c | 14 ++++++++++++++ source/dnode/vnode/src/vnd/vnodeSync.c | 14 ++++++++++++++ source/libs/sync/src/syncMain.c | 12 +++++++++++- 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 35e79cf45d..aa563343f8 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -129,6 +129,9 @@ typedef struct SSyncFSM { void (*FpReConfigCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta); void (*FpLeaderTransferCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); + void (*FpBecomeLeaderCb)(struct SSyncFSM* pFsm); + void (*FpBecomeFollowerCb)(struct SSyncFSM* pFsm); + int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot, void* pReaderParam, void** ppReader); int32_t (*FpGetSnapshotInfo)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index b8ff7be46a..37d5aeb62d 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -166,6 +166,18 @@ void mndLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cb mDebug("vgId:1, mnode leader transfer finish"); } +static void mndBecomeFollower(struct SSyncFSM *pFsm) { + SMnode *pMnode = pFsm->data; + mDebug("vgId:1, become follower"); + + // clear old leader resource +} + +static void mndBecomeLeader(struct SSyncFSM *pFsm) { + SMnode *pMnode = pFsm->data; + mDebug("vgId:1, become leader"); +} + SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pMnode; @@ -175,6 +187,8 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { pFsm->FpRestoreFinishCb = mndRestoreFinish; pFsm->FpLeaderTransferCb = mndLeaderTransfer; pFsm->FpReConfigCb = mndReConfig; + pFsm->FpBecomeLeaderCb = mndBecomeLeader; + pFsm->FpBecomeFollowerCb = mndBecomeFollower; pFsm->FpGetSnapshot = mndSyncGetSnapshot; pFsm->FpGetSnapshotInfo = mndSyncGetSnapshotInfo; pFsm->FpSnapshotStartRead = mndSnapshotStartRead; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index c525d09057..7ac124fdd3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -672,6 +672,18 @@ static void vnodeRestoreFinish(struct SSyncFSM *pFsm) { vDebug("vgId:%d, sync restore finished", pVnode->config.vgId); } +static void vnodeBecomeFollower(struct SSyncFSM *pFsm) { + SVnode *pVnode = pFsm->data; + vDebug("vgId:%d, become follower", pVnode->config.vgId); + + // clear old leader resource +} + +static void vnodeBecomeLeader(struct SSyncFSM *pFsm) { + SVnode *pVnode = pFsm->data; + vDebug("vgId:%d, become leader", pVnode->config.vgId); +} + static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pVnode; @@ -681,6 +693,8 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshot; pFsm->FpRestoreFinishCb = vnodeRestoreFinish; pFsm->FpLeaderTransferCb = vnodeLeaderTransfer; + pFsm->FpBecomeLeaderCb = vnodeBecomeLeader; + pFsm->FpBecomeFollowerCb = vnodeBecomeFollower; pFsm->FpReConfigCb = vnodeSyncReconfig; pFsm->FpSnapshotStartRead = vnodeSnapshotStartRead; pFsm->FpSnapshotStopRead = vnodeSnapshotStopRead; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 20b4b23bf7..5f3ff3015c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2028,6 +2028,11 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // reset elect timer syncNodeResetElectTimer(pSyncNode); + // call back + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpBecomeFollowerCb != NULL) { + pSyncNode->pFsm->FpBecomeFollowerCb(pSyncNode->pFsm); + } + // trace log do { int32_t debugStrLen = strlen(debugStr); @@ -2109,6 +2114,11 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // start heartbeat timer syncNodeStartHeartbeatTimer(pSyncNode); + // call back + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpBecomeLeaderCb != NULL) { + pSyncNode->pFsm->FpBecomeLeaderCb(pSyncNode->pFsm); + } + // trace log do { int32_t debugStrLen = strlen(debugStr); @@ -3100,7 +3110,7 @@ void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntries syncNodeEventLog(pSyncNode, logBuf); } - void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { +void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); From 95f696b30db2a7a082c8f29adeeba2dbaede205b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 10 Aug 2022 17:23:07 +0800 Subject: [PATCH 77/96] enh(tmq): prevent drop db when topic exists --- examples/rust | 1 - include/util/taoserror.h | 2 ++ source/client/src/tmq.c | 21 +++++---------------- source/dnode/mnode/impl/inc/mndTopic.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 10 ++++++---- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndTopic.c | 20 ++++++++++++++++++++ source/util/src/terror.c | 1 + tools/taos-tools | 1 - 9 files changed, 36 insertions(+), 23 deletions(-) delete mode 160000 examples/rust delete mode 160000 tools/taos-tools diff --git a/examples/rust b/examples/rust deleted file mode 160000 index 7ed7a97715..0000000000 --- a/examples/rust +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7ed7a97715388fa144718764d6bf20f9bfc29a12 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ce73dceec2..3ca6978156 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -284,11 +284,13 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_CONSUMER_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_TOPIC_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x03EB) #define TSDB_CODE_MND_CGROUP_USED TAOS_DEF_ERROR_CODE(0, 0x03EC) +#define TSDB_CODE_MND_TOPIC_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03ED) // mnode-stream #define TSDB_CODE_MND_STREAM_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F0) #define TSDB_CODE_MND_STREAM_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F1) #define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2) +#define TSDB_CODE_MND_STREAM_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03F3) // mnode-sma #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index c4375c85b4..c0874caf95 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -204,7 +204,7 @@ typedef struct { typedef struct { SMqCommitCbParamSet* params; STqOffset* pOffset; -} SMqCommitCbParam2; +} SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); @@ -368,8 +368,8 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { return sprintf(dst, "%s:%d", topicName, vg); } -int32_t tmqCommitCb2(void* param, SDataBuf* pBuf, int32_t code) { - SMqCommitCbParam2* pParam = (SMqCommitCbParam2*)param; +int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { + SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; // push into array #if 0 @@ -440,7 +440,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT tEncodeSTqOffset(&encoder, pOffset); // build param - SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); + SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); pParam->params = pParamSet; pParam->pOffset = pOffset; @@ -465,7 +465,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT pMsgSendInfo->requestObjRefId = 0; pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; - pMsgSendInfo->fp = tmqCommitCb2; + pMsgSendInfo->fp = tmqCommitCb; pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; // send msg @@ -1313,17 +1313,6 @@ int32_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { #endif SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { - /*int64_t reqOffset;*/ - /*if (pVg->currentOffset >= 0) {*/ - /*reqOffset = pVg->currentOffset;*/ - /*} else {*/ - /*if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {*/ - /*tscError("unable to poll since no committed offset but reset offset is set to none");*/ - /*return NULL;*/ - /*}*/ - /*reqOffset = tmq->resetOffsetCfg;*/ - /*}*/ - SMqPollReq* pReq = taosMemoryCalloc(1, sizeof(SMqPollReq)); if (pReq == NULL) { return NULL; diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index c5c4800e02..9f516904ce 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -32,6 +32,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic); SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw); int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); +int32_t mndCheckTopicExist(SMnode *pMnode, SDbObj *pDb); const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 89ea7abc23..9dde083f50 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -995,11 +995,13 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); mndTransSetDbName(pTrans, pDb->name, NULL); + if (mndCheckTopicExist(pMnode, pDb) < 0) goto _OVER; + if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER; - if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER; - if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER; - if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER; + /*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ + /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ + /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; @@ -1706,7 +1708,7 @@ static void setPerfSchemaDbCfg(SDbObj *pDbObj) { static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; int32_t *numOfTables = p1; - int64_t uid = *(int64_t*)p2; + int64_t uid = *(int64_t *)p2; if (pVgroup->dbUid == uid) { *numOfTables += pVgroup->numOfTables; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7bde05aca5..0f20188033 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -837,7 +837,7 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { sdbCancelFetch(pSdb, pIter); mError("db:%s, failed to drop stream:%s since sourceDbUid:%" PRId64 " not match with targetDbUid:%" PRId64, pDb->name, pStream->name, pStream->sourceDbUid, pStream->targetDbUid); - terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; + terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; return -1; } else { #if 0 diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index f343c2e055..820bb4b636 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -782,6 +782,26 @@ static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } +int32_t mndCheckTopicExist(SMnode *pMnode, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + SMqTopicObj *pTopic = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); + if (pIter == NULL) break; + + if (pTopic->dbUid == pDb->uid) { + sdbRelease(pSdb, pTopic); + terrno = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED; + return -1; + } + + sdbRelease(pSdb, pTopic); + } + return 0; +} + int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t code = 0; SSdb *pSdb = pMnode->pSdb; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index f0a58c9bdc..bb26f9b2f7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -286,6 +286,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST, "Subcribe not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_OFFSET_NOT_EXIST, "Offset not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_SUBSCRIBED, "Topic subscribed cannot be dropped") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_MUST_BE_DELETED, "Topic must be dropped first") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being used by some consumer") // mnode-stream diff --git a/tools/taos-tools b/tools/taos-tools deleted file mode 160000 index 3c7dafeea3..0000000000 --- a/tools/taos-tools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3c7dafeea3e558968165b73bee0f51024898e3da From 1d6d0aa2db2ffc8024f27a5145b6811f4469f879 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 10 Aug 2022 17:26:44 +0800 Subject: [PATCH 78/96] enh: row iter and fetch optimization --- include/common/tdataformat.h | 2 - include/common/trow.h | 13 +- source/common/src/trow.c | 189 +++++++------------------ source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/libs/executor/src/tsimplehash.c | 79 +++++------ source/libs/parser/src/parInsert.c | 4 +- 7 files changed, 108 insertions(+), 183 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 311eb72b4d..3679b3773b 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -200,8 +200,6 @@ struct STag { #if 1 //================================================================================================================================================ // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP -#define TD_SUPPORT_READ2 -#define TD_SUPPORT_BACK2 // suppport back compatibility of 2.0 #define TASSERT(x) ASSERT(x) diff --git a/include/common/trow.h b/include/common/trow.h index bd8510dde6..a1db2ef708 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -195,6 +195,12 @@ static FORCE_INLINE void tdRowSetVal(SCellVal *pVal, uint8_t valType, void *val) pVal->valType = valType; pVal->val = val; } +/** + * @brief Primary TS column not included. + * + * @param pRow + * @return FORCE_INLINE + */ static FORCE_INLINE col_id_t tdRowGetNCols(STSRow *pRow) { return *(col_id_t *)TD_ROW_NCOLS(pRow); } static FORCE_INLINE void tdRowCpy(void *dst, const STSRow *pRow) { memcpy(dst, pRow, TD_ROW_LEN(pRow)); } static FORCE_INLINE const char *tdRowEnd(STSRow *pRow) { return (const char *)POINTER_SHIFT(pRow, TD_ROW_LEN(pRow)); } @@ -250,7 +256,7 @@ static FORCE_INLINE void *tdGetBitmapAddrTp(STSRow *pRow, uint32_t flen) { static FORCE_INLINE void *tdGetBitmapAddrKv(STSRow *pRow, col_id_t nKvCols) { // The primary TS key is stored separatedly and is Norm value, thus should minus 1 firstly - return POINTER_SHIFT(TD_ROW_COL_IDX(pRow), (--nKvCols) * sizeof(SKvRowIdx)); + return POINTER_SHIFT(TD_ROW_COL_IDX(pRow), nKvCols * sizeof(SKvRowIdx)); } void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols); int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); @@ -322,10 +328,9 @@ typedef struct { void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema); int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow); -bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); -bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal); -bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); +bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal); +bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal); bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, SCellVal *pVal); bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6f58a0ded1..b13e5f1ca8 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -330,15 +330,17 @@ void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterReset(&iter, row); printf("%s >>>type:%d,sver:%d ", tag, (int32_t)TD_ROW_TYPE(row), (int32_t)TD_ROW_SVER(row)); - for (int i = 0; i < pSchema->numOfCols; ++i) { - STColumn *stCol = pSchema->columns + i; - SCellVal sVal = {255, NULL}; - if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { + + STColumn *cols = (STColumn *)&iter.pSchema->columns; + while (true) { + SCellVal sVal = {.valType = 255, NULL}; + if (!tdSTSRowIterNext(&iter, &sVal)) { break; } ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); - tdSCellValPrint(&sVal, stCol->type); + tdSCellValPrint(&sVal, cols[iter.colIdx - 1].type); } + printf("\n"); } @@ -425,7 +427,7 @@ bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVa tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); return true; } - int16_t nCols = tdRowGetNCols(pRow) - 1; + int16_t nCols = tdRowGetNCols(pRow); if (nCols <= 0) { pVal->valType = TD_VTYPE_NONE; return true; @@ -456,46 +458,41 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } -bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { + if (pIter->colIdx >= pIter->pSchema->numOfCols) { + return false; + } + + STColumn *pCol = &pIter->pSchema->columns[pIter->colIdx]; + + if (pCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { pVal->val = &pIter->pRow->ts; pVal->valType = TD_VTYPE_NORM; + ++pIter->colIdx; return true; } if (TD_IS_TP_ROW(pIter->pRow)) { - STColumn *pCol = NULL; - STSchema *pSchema = pIter->pSchema; - while (pIter->colIdx < pSchema->numOfCols) { - pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key - if (colId == pCol->colId) { - break; - } else if (pCol->colId < colId) { - ++pIter->colIdx; - continue; - } else { - return false; - } - } tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); - ++pIter->colIdx; } else if (TD_IS_KV_ROW(pIter->pRow)) { - return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal); + tdGetKvRowValOfColEx(pIter, pCol->colId, &pIter->kvIdx, pVal); } else { - pVal->valType = TD_VTYPE_NONE; - terrno = TSDB_CODE_INVALID_PARA; - if (COL_REACH_END(colId, pIter->maxColId)) return false; + ASSERT(0); } + ++pIter->colIdx; + return true; } -bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) { +bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal) { STSRow *pRow = pIter->pRow; SKvRowIdx *pKvIdx = NULL; bool colFound = false; - col_id_t kvNCols = tdRowGetNCols(pRow) - 1; + col_id_t kvNCols = tdRowGetNCols(pRow); + void *pColIdx = TD_ROW_COL_IDX(pRow); + while (*nIdx < kvNCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx)); + pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, *nIdx * sizeof(SKvRowIdx)); if (pKvIdx->colId == colId) { ++(*nIdx); pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); @@ -518,34 +515,37 @@ bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, } } -#ifdef TD_SUPPORT_BITMAP - int16_t colIdx = -1; - if (pKvIdx) colIdx = POINTER_DISTANCE(pKvIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); - if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + if (tdGetBitmapValType(pIter->pBitmap, (*nIdx) - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif return true; } bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { STSRow *pRow = pIter->pRow; - if (IS_VAR_DATA_TYPE(colType)) { - pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + if (pRow->statis == 0) { + pVal->valType = TD_VTYPE_NORM; + if (IS_VAR_DATA_TYPE(colType)) { + pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + return TSDB_CODE_SUCCESS; } -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; + return terrno; + } + + if (pVal->valType == TD_VTYPE_NORM) { + if (IS_VAR_DATA_TYPE(colType)) { + pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif return true; } @@ -625,7 +625,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { if (maxVarDataLen > 0) { varBuf = taosMemoryMalloc(maxVarDataLen); if (!varBuf) { - if(isAlloc) { + if (isAlloc) { taosMemoryFreeClear(*ppRow); } terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -673,45 +673,6 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { return 0; } -bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pVal->val = &pIter->pRow->ts; - pVal->valType = TD_VTYPE_NORM; - return true; - } - - STSRow *pRow = pIter->pRow; - int16_t colIdx = -1; - if (TD_IS_TP_ROW(pRow)) { - STSchema *pSchema = pIter->pSchema; - STColumn *pCol = - (STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ); - if (!pCol) { - pVal->valType = TD_VTYPE_NONE; - if (COL_REACH_END(colId, pIter->maxColId)) return false; - return true; - } -#ifdef TD_SUPPORT_BITMAP - colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); -#endif - tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); - } else if (TD_IS_KV_ROW(pRow)) { - SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), - compareKvRowColId, TD_EQ); -#ifdef TD_SUPPORT_BITMAP - if (pIdx) { - colIdx = POINTER_DISTANCE(pIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); - } -#endif - tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); - } else { - if (COL_REACH_END(colId, pIter->maxColId)) return false; - pVal->valType = TD_VTYPE_NONE; - } - - return true; -} - static int32_t tdCompareColId(const void *arg1, const void *arg2) { int32_t colId = *(int32_t *)arg1; STColumn *pCol = (STColumn *)arg2; @@ -853,7 +814,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP - TASSERT(colIdx < tdRowGetNCols(pRow) - 1); + TASSERT(colIdx < tdRowGetNCols(pRow)); if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { output->valType = TD_VTYPE_NONE; return terrno; @@ -938,7 +899,7 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp break; case TD_VTYPE_NONE: if (!pBuilder->hasNone) pBuilder->hasNone = true; - break; + return TSDB_CODE_SUCCESS; default: ASSERT(0); break; @@ -970,13 +931,11 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo STSRow *row = pBuilder->pBuf; // No need to store None/Null values. + SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); + pColIdx->colId = colId; + pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN if (valType == TD_VTYPE_NORM) { - // ts key stored in STSRow.ts - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); if (IS_VAR_DATA_TYPE(colType)) { if (isCopyVarData) { memcpy(ptr, val, varDataTLen(val)); @@ -987,26 +946,6 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo TD_ROW_LEN(row) += TYPE_BYTES[colType]; } } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - const void *nullVal = getNullValue(colType); - - if (IS_VAR_DATA_TYPE(colType)) { - if (isCopyVarData) { - memcpy(ptr, nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(ptr, nullVal, TYPE_BYTES[colType]); - TD_ROW_LEN(row) += TYPE_BYTES[colType]; - } - } -#endif return 0; } @@ -1044,24 +983,6 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), val, TYPE_BYTES[colType]); } } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - // TODO: Null value for new data types imported since 3.0 need to be defined. - const void *nullVal = getNullValue(colType); - if (IS_VAR_DATA_TYPE(colType)) { - // ts key stored in STSRow.ts - *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); - - if (isCopyVarData) { - memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), nullVal, TYPE_BYTES[colType]); - } - } -#endif return 0; } @@ -1133,14 +1054,14 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { break; case TD_ROW_KV: #ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols - 1); memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps); #endif len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + pBuilder->nBoundBitmaps; // add TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); - TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); + TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols - 1); break; default: TASSERT(0); @@ -1329,7 +1250,7 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { pIter->pRow = pRow; pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); pIter->offset = 0; - pIter->colIdx = PRIMARYKEY_TIMESTAMP_COL_ID; + pIter->colIdx = 0; pIter->kvIdx = 0; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 6ce8dbe5d9..ced98c8afa 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -325,7 +325,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { for (int32_t i = 0; i < colActual; i++) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SCellVal sVal = {0}; - if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { + if (!tdSTSRowIterNext(&iter, &sVal)) { break; } if (colDataAppend(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 468370b1a7..1da65288d0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -182,7 +182,7 @@ static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes); - tsdbInfo("-------------------%d\n", pCol->info.bytes); + // tsdbInfo("-------------------%d\n", pCol->info.bytes); } } diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 8f7ff04d92..e709643af9 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -13,38 +13,37 @@ * along with this program. If not, see . */ -#include "os.h" #include "tsimplehash.h" +#include "os.h" #include "taoserror.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 -#define HASH_MAX_CAPACITY (1024*1024*16) +#define HASH_MAX_CAPACITY (1024 * 1024 * 16) #define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) -#define GET_SHASH_NODE_KEY(_n, _dl) ((char*)(_n) + sizeof(SHNode) + (_dl)) -#define GET_SHASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHNode)) +#define GET_SHASH_NODE_KEY(_n, _dl) ((char *)(_n) + sizeof(SHNode) + (_dl)) +#define GET_SHASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHNode)) -#define HASH_INDEX(v, c) ((v) & ((c)-1)) -#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) +#define HASH_INDEX(v, c) ((v) & ((c)-1)) -#define FREE_HASH_NODE(_n) \ - do { \ - taosMemoryFreeClear(_n); \ +#define FREE_HASH_NODE(_n) \ + do { \ + taosMemoryFreeClear(_n); \ } while (0); typedef struct SHNode { - struct SHNode *next; - char data[]; + struct SHNode *next; + char data[]; } SHNode; struct SSHashObj { - SHNode **hashList; - size_t capacity; // number of slots - int64_t size; // number of elements in hash table - _hash_fn_t hashFp; // hash function - _equal_fn_t equalFp; // equal function - int32_t keyLen; - int32_t dataLen; + SHNode **hashList; + size_t capacity; // number of slots + int64_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function + int32_t keyLen; + int32_t dataLen; }; static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { @@ -62,7 +61,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t capacity = 4; } - SSHashObj* pHashObj = (SSHashObj*) taosMemoryCalloc(1, sizeof(SSHashObj)); + SSHashObj *pHashObj = (SSHashObj *)taosMemoryCalloc(1, sizeof(SSHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -72,7 +71,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t pHashObj->capacity = taosHashCapacity((int32_t)capacity); pHashObj->equalFp = memcmp; - pHashObj->hashFp = fn; + pHashObj->hashFp = fn; ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); pHashObj->keyLen = keyLen; @@ -91,7 +90,7 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { if (pHashObj == NULL) { return 0; } - return (int32_t)atomic_load_64((int64_t*)&pHashObj->size); + return (int32_t)atomic_load_64((int64_t *)&pHashObj->size); } static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { @@ -108,41 +107,42 @@ static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pDat } static void taosHashTableResize(SSHashObj *pHashObj) { - if (!HASH_NEED_RESIZE(pHashObj)) { + if (!SHASH_NEED_RESIZE(pHashObj)) { return; } int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); if (newCapacity > HASH_MAX_CAPACITY) { -// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", -// pHashObj->capacity, HASH_MAX_CAPACITY); + // uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", + // pHashObj->capacity, HASH_MAX_CAPACITY); return; } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { -// qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); + // qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; } size_t inc = newCapacity - pHashObj->capacity; - memset((char*)pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc); + memset((char *)pNewEntryList + pHashObj->capacity * sizeof(void *), 0, inc); pHashObj->hashList = pNewEntryList; pHashObj->capacity = newCapacity; for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { - SHNode* pNode = pHashObj->hashList[idx]; - SHNode *pNext; - SHNode *pPrev = NULL; - + SHNode *pNode = pHashObj->hashList[idx]; if (pNode == NULL) { continue; } + SHNode *pNext; + SHNode *pPrev = NULL; + + while (pNode != NULL) { - void* key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen); + void *key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen); uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->dataLen); int32_t newIdx = HASH_INDEX(hashVal, pHashObj->capacity); @@ -166,8 +166,9 @@ static void taosHashTableResize(SSHashObj *pHashObj) { int64_t et = taosGetTimestampUs(); -// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, -// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); + // uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", + // (int32_t)pHashObj->capacity, + // ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { @@ -210,7 +211,7 @@ int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { pNewNode->next = pHashObj->hashList[slot]; pHashObj->hashList[slot] = pNewNode; atomic_add_fetch_64(&pHashObj->size, 1); - } else { //update data + } else { // update data memcpy(GET_SHASH_NODE_DATA(pNode), data, pHashObj->dataLen); } @@ -230,9 +231,7 @@ static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void return pNode; } -static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) { - return tSimpleHashGetSize(pHashObj) == 0; -} +static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) { return tSimpleHashGetSize(pHashObj) == 0; } void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL) { @@ -299,9 +298,9 @@ size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { return (pHashObj->capacity * sizeof(void *)) + sizeof(SHNode) * tSimpleHashGetSize(pHashObj) + sizeof(SSHashObj); } -void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen) { +void *tSimpleHashGetKey(const SSHashObj *pHashObj, void *data, size_t *keyLen) { int32_t offset = offsetof(SHNode, data); - SHNode *node = ((SHNode*)(char*)data - offset); + SHNode *node = ((SHNode *)(char *)data - offset); if (keyLen != NULL) { *keyLen = pHashObj->keyLen; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 9c9807f843..b7532173c8 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1192,7 +1192,10 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, pBuilder->hasNone = true; } + tdSRowEnd(pBuilder); + *gotRow = true; + #ifdef TD_DEBUG_PRINT_ROW STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1); tdSRowPrint(row, pSTSchema, __func__); @@ -1201,7 +1204,6 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, } // *len = pBuilder->extendedRowSize; - tdSRowEnd(pBuilder); return TSDB_CODE_SUCCESS; } From 4cd1452e61843ea9fd1e3735538d67d1c8efd08b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 10 Aug 2022 17:55:04 +0800 Subject: [PATCH 79/96] fix(stream): check udf --- source/libs/executor/src/timewindowoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9198862ba3..610303dc50 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1695,7 +1695,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { for (int32_t i = 0; i < numOfCols; i++) { - if (!fmIsInvertible(pFCtx[i].functionId)) { + if (fmIsUserDefinedFunc(pFCtx[i].functionId) || !fmIsInvertible(pFCtx[i].functionId)) { return false; } } From 0801dc4f1d4de4659576fac1de2fd0770276e1df Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 10 Aug 2022 17:59:02 +0800 Subject: [PATCH 80/96] doc: add performance_schema page --- docs/zh/12-taos-sql/14-stream.md | 12 --- docs/zh/12-taos-sql/23-perf.md | 129 +++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 12 deletions(-) create mode 100644 docs/zh/12-taos-sql/23-perf.md diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 7ff7da2bfb..024742ed8b 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -89,10 +89,6 @@ T = 最新事件时间 - watermark 无论在哪种模式下，watermark 都应该被妥善设置，来得到正确结果（直接丢弃模式）或避免频繁触发重算带来的性能开销（重新计算模式）。 -## 流式计算的数据填充策略 - -TODO - ## 流式计算与会话窗口（session window） ```sql @@ -105,14 +101,6 @@ window_clause: { 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 -## 流式计算的监控与流任务分布查询 - -TODO - -## 流式计算的内存控制与存算分离 - -TODO - ## 流式计算的暂停与恢复 ```sql diff --git a/docs/zh/12-taos-sql/23-perf.md b/docs/zh/12-taos-sql/23-perf.md new file mode 100644 index 0000000000..4c7660e829 --- /dev/null +++ b/docs/zh/12-taos-sql/23-perf.md @@ -0,0 +1,129 @@ +--- +sidebar_label: 性能数据库 +title: 性能数据库 +--- + +TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其中存储了与性能有关的统计数据。本节详细介绍其中的表和详细的表结构。 + +## PERF_APP + +提供接入集群的应用（客户端）的相关信息。也可以使用 SHOW APPS 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | ------------------------------- | +| 1 | app_id | UBIGINT | 客户端 ID | +| 2 | ip | BINARY(16) | 客户端地址 | +| 3 | pid | INT | 客户端进程 号 | +| 4 | name | BINARY(24) | 客户端名称 | +| 5 | start_time | TIMESTAMP | 客户端启动时间 | +| 6 | insert_req | UBIGINT | insert 请求次数 | +| 7 | insert_row | UBIGINT | insert 插入行数 | +| 8 | insert_time | UBIGINT | insert 请求的处理时间，单位微秒 | +| 9 | insert_bytes | UBIGINT | insert 请求消息字节数 | +| 10 | fetch_bytes | UBIGINT | 查询结果字节数 | +| 11 | query_time | UBIGINT | 查询请求处理时间 | +| 12 | slow_query | UBIGINT | 慢查询（处理时间 >= 3 秒）个数 | +| 13 | total_req | UBIGINT | 总请求数 | +| 14 | current_req | UBIGINT | 当前正在处理的请求个数 | +| 15 | last_access | TIMESTAMP | 最后更新时间 | + +## PERF_CONNECTIONS + +数据库的连接的相关信息。也可以使用 SHOW CONNECTIONS 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | -------------------------------------------------- | +| 1 | conn_id | INT | 连接 ID | +| 2 | user | BINARY(24) | 用户名 | +| 3 | app | BINARY(24) | 客户端名称 | +| 4 | pid | UINT | 发起此连接的客户端在自己所在服务器或主机上的进程号 | +| 5 | end_point | BINARY(128) | 客户端地址 | +| 6 | login_time | TIMESTAMP | 登录时间 | +| 7 | last_access | TIMESTAMP | 最后更新时间 | + +## PERF_QUERIES + +提供当前正在执行的 SQL 语句的信息。也可以使用 SHOW QUERIES 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | ---------------------------- | +| 1 | kill_id | UBIGINT | 用来停止查询的 ID | +| 2 | query_id | INT | 查询 ID | +| 3 | conn_id | UINT | 连接 ID | +| 4 | app | BINARY(24) | app 名称 | +| 5 | pid | INT | app 在自己所在主机上的进程号 | +| 6 | user | BINARY(24) | 用户名 | +| 7 | end_point | BINARY(16) | 客户端地址 | +| 8 | create_time | TIMESTAMP | 创建时间 | +| 9 | exec_usec | BIGINT | 已执行时间 | +| 10 | stable_query | BOOL | 是否是超级表查询 | +| 11 | sub_num | INT | 子查询数量 | +| 12 | sub_status | BINARY(1000) | 子查询状态 | +| 13 | sql | BINARY(1024) | SQL 语句 | + +## PERF_TOPICS + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ------------------------------ | +| 1 | topic_name | BINARY(192) | topic 名称 | +| 2 | db_name | BINARY(64) | topic 相关的 DB | +| 3 | create_time | TIMESTAMP | topic 的 创建时间 | +| 4 | sql | BINARY(1024) | 创建该 topic 时所用的 SQL 语句 | + +## PERF_CONSUMERS + +| # | **列名** | **数据类型** | **说明** | +| --- | :------------: | ------------ | ----------------------------------------------------------- | +| 1 | consumer_id | BIGINT | 消费者的唯一 ID | +| 2 | consumer_group | BINARY(192) | 消费者组 | +| 3 | client_id | BINARY(192) | 用户自定义字符串，通过创建 consumer 时指定 client_id 来展示 | +| 4 | status | BINARY(20) | 消费者当前状态 | +| 5 | topics | BINARY(204) | 被订阅的 topic。若订阅多个 topic，则展示为多行 | +| 6 | up_time | TIMESTAMP | 第一次连接 taosd 的时间 | +| 7 | subscribe_time | TIMESTAMP | 上一次发起订阅的时间 | +| 8 | rebalance_time | TIMESTAMP | 上一次触发 rebalance 的时间 | + +## PERF_SUBSCRIPTIONS + +| # | **列名** | **数据类型** | **说明** | +| --- | :------------: | ------------ | ------------------------ | +| 1 | topic_name | BINARY(204) | 被订阅的 topic | +| 2 | consumer_group | BINARY(193) | 订阅者的消费者组 | +| 3 | vgroup_id | INT | 消费者被分配的 vgroup id | +| 4 | consumer_id | BIGINT | 消费者的唯一 id | + +## PERF_TRANS + +| # | **列名** | **数据类型** | **说明** | +| --- | :--------------: | ------------ | -------- | +| 1 | id | INT | | +| 2 | create_time | TIMESTAMP | | +| 3 | stage | BINARY(12) | | +| 4 | db1 | BINARY(64) | | +| 5 | db2 | BINARY(64) | | +| 6 | failed_times | INT | | +| 7 | last_exec_time | TIMESTAMP | | +| 8 | last_action_info | BINARY(511) | | + +## PERF_SMAS + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ------------------------------------------- | +| 1 | sma_name | BINARY(192) | 时间维度的预计算 (time-range-wise sma) 名称 | +| 2 | create_time | TIMESTAMP | sma 创建时间 | +| 3 | stable_name | BINARY(192) | sma 所属的超级表名称 | +| 4 | vgroup_id | INT | sma 专属的 vgroup 名称 | + +## PERF_STREAMS + +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | --------------------------------------- | +| 1 | stream_name | BINARY(64) | 流计算名称 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | sql | BINARY(1024) | 创建流计算时提供的 SQL 语句 | +| 4 | status | BIANRY(20) | 流当前状态 | +| 5 | source_db | BINARY(64) | 源数据库 | +| 6 | target_db | BIANRY(64) | 目的数据库 | +| 7 | target_table | BINARY(192) | 流计算写入的目标表 | +| 8 | watermark | BIGINT | watermark，详见 SQL 手册流式计算 | +| 9 | trigger | INT | 计算结果推送模式，详见 SQL 手册流式计算 | From 668a78a79f367f83bf8ee2c3e238925ff5350194 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 10 Aug 2022 18:02:43 +0800 Subject: [PATCH 81/96] fix: fix histogram not setting content when process all null columns --- source/libs/function/src/builtinsimpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 9aa7c11fce..b325848f11 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -4281,9 +4281,9 @@ static int32_t histogramFunctionImpl(SqlFunctionCtx* pCtx, bool isPartial) { } if (!isPartial) { - SET_VAL(GET_RES_INFO(pCtx), numOfElems, pInfo->numOfBins); + GET_RES_INFO(pCtx)->numOfRes = pInfo->numOfBins; } else { - SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + GET_RES_INFO(pCtx)->numOfRes = 1; } return TSDB_CODE_SUCCESS; } From 9e5d3473832152ee4637ae3952de880f27641b5a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 10 Aug 2022 18:22:22 +0800 Subject: [PATCH 82/96] enh(tmq): enable background heartbeat by default --- source/client/src/tmq.c | 1 + source/dnode/mnode/impl/src/mndStream.c | 18 +++++++++++++++--- source/util/src/terror.c | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index c0874caf95..28c359b2fb 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -212,6 +212,7 @@ tmq_conf_t* tmq_conf_new() { conf->autoCommit = true; conf->autoCommitInterval = 5000; conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; + conf->hbBgEnable = true; return conf; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0f20188033..b172b00fab 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -929,14 +929,26 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->status, true); + char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pStream->sourceDb, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&n, varDataVal(sourceDB)); + varDataSetLen(sourceDB, strlen(varDataVal(sourceDB))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->sourceDb, true); + colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false); + char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pStream->targetDb, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&n, varDataVal(targetDB)); + varDataSetLen(targetDB, strlen(varDataVal(targetDB))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetDb, true); + colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false); + char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n)); + varDataSetLen(targetSTB, strlen(varDataVal(targetSTB))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetSTbName, true); + colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index bb26f9b2f7..8a3d0620ed 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -281,7 +281,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC, "Invalid topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_QUERY, "Topic with invalid query") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION, "Topic with invalid option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_EXIST, "Consumer not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED, "Consumer unchanged") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED, "Topic unchanged") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST, "Subcribe not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_OFFSET_NOT_EXIST, "Offset not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer not ready") From 365dc43fec708e6f26e2cfbc2e293510f69ec073 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 10 Aug 2022 18:53:29 +0800 Subject: [PATCH 83/96] enh(tmq): add log --- source/dnode/mnode/impl/src/mndSubscribe.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 8feed476cd..3f310ee9c0 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -398,13 +398,27 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } } - // 8. TODO generate logs - mInfo("rebalance calculation completed, rebalanced vg:"); + // 8. generate logs + mInfo("mq rebalance: calculation completed, rebalanced vg:"); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); - mInfo("vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId, + mInfo("mq rebalance: vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } + { + void *pIter = NULL; + while (1) { + pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); + if (pIter == NULL) break; + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; + int32_t sz = taosArrayGetSize(pConsumerEp->vgs); + mInfo("mq rebalance: final cfg: consumer %ld has %d vg", pConsumerEp->consumerId, sz); + for (int32_t i = 0; i < sz; i++) { + SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); + mInfo("mq rebalance: final cfg: vg %d to consumer %ld", pVgEp->vgId, pConsumerEp->consumerId); + } + } + } // 9. clear taosHashCleanup(pHash); From bed9ca4d0854c21947b6ee862bbd7055bfeccf32 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 19:37:10 +0800 Subject: [PATCH 84/96] doc: change 3.0 doc --- docs/en/14-reference/08-taos-shell.md | 35 +++++++++-------- .../14-reference/09-support-platform/index.md | 11 +++--- docs/en/14-reference/12-directory.md | 1 - docs/zh/14-reference/08-taos-shell.md | 39 ++++++++++--------- .../14-reference/09-support-platform/index.md | 17 +++----- docs/zh/14-reference/12-directory.md | 1 - 6 files changed, 48 insertions(+), 56 deletions(-) diff --git a/docs/en/14-reference/08-taos-shell.md b/docs/en/14-reference/08-taos-shell.md index 002b515093..c947e86d1c 100644 --- a/docs/en/14-reference/08-taos-shell.md +++ b/docs/en/14-reference/08-taos-shell.md @@ -47,27 +47,28 @@ If the displayed content is followed by `...` you can use this command to change You can change the behavior of TDengine CLI by specifying command-line parameters. The following parameters are commonly used. -- -h, --host=HOST: FQDN of the server where the TDengine server is to be connected. Default is to connect to the local service -- -P, --port=PORT: Specify the port number to be used by the server. Default is `6030` -- -u, --user=USER: the user name to use when connecting. Default is `root` -- -p, --password=PASSWORD: the password to use when connecting to the server. Default is `taosdata` +- -h HOST: FQDN of the server where the TDengine server is to be connected. Default is to connect to the local service +- -P PORT: Specify the port number to be used by the server. Default is `6030` +- -u USER: the user name to use when connecting. Default is `root` +- -p PASSWORD: the password to use when connecting to the server. Default is `taosdata` - -?, --help: print out all command-line arguments And many more parameters. -- -c, --config-dir: Specify the directory where configuration file exists. The default is `/etc/taos`, and the default name of the configuration file in this directory is `taos.cfg` -- -C, --dump-config: Print the configuration parameters of `taos.cfg` in the default directory or specified by -c -- -d, --database=DATABASE: Specify the database to use when connecting to the server -- -D, --directory=DIRECTORY: Import the SQL script file in the specified path -- -f, --file=FILE: Execute the SQL script file in non-interactive mode -- -k, --check=CHECK: Specify the table to be checked -- -l, --pktlen=PKTLEN: Test package size to be used for network testing -- -n, --netrole=NETROLE: test scope for network connection test, default is `startup`. The value can be `client`, `server`, `rpc`, `startup`, `sync`, `speed`, or `fqdn`. -- -r, --raw-time: output the timestamp format as unsigned 64-bits integer (uint64_t in C language) -- -s, --commands=COMMAND: execute SQL commands in non-interactive mode -- -S, --pkttype=PKTTYPE: Specify the packet type used for network testing. The default is TCP, can be specified as either TCP or UDP when `speed` is specified to `netrole` parameter -- -T, --thread=THREADNUM: The number of threads to import data in multi-threaded mode -- -s, --commands: Run TDengine CLI commands without entering the terminal +- -a AUTHSTR: The auth string to use when connecting to the server +- -A: Generate auth string from password +- -c CONFIGDIR: Specify the directory where configuration file exists. The default is `/etc/taos`, and the default name of the configuration file in this directory is `taos.cfg` +- -C: Print the configuration parameters of `taos.cfg` in the default directory or specified by -c +- -d DATABASE: Specify the database to use when connecting to the server +- -f FILE: Execute the SQL script file in non-interactive mode +- -k: Check the service status, 0: unavailable，1: network ok，2: service ok，3: service degraded，4: exiting +- -l PKTLEN: Test package length to be used for network testing +- -n NETROLE: test scope for network connection test, default is `client`. The value can be `client`, `server` +- -N PKTNUM: Test package numbers to be used for network testing +- -r: output the timestamp format as unsigned 64-bits integer (uint64_t in C language) +- -s COMMAND: execute SQL commands in non-interactive mode +- -t: Check the details of the service status，status same as -k +- -w DISPLAYWIDTH: 客户端列显示宽度 - -z, --timezone=TIMEZONE: Specify time zone. Default is the value of current configuration file - -V, --version: Print out the current version number diff --git a/docs/en/14-reference/09-support-platform/index.md b/docs/en/14-reference/09-support-platform/index.md index af15758225..656344eff4 100644 --- a/docs/en/14-reference/09-support-platform/index.md +++ b/docs/en/14-reference/09-support-platform/index.md @@ -5,12 +5,11 @@ description: "List of platforms supported by TDengine server, client, and connec ## List of supported platforms for TDengine server -| | **CentOS 7/8** | **Ubuntu 16/18/20** | **Other Linux** | -| ------------ | -------------- | ------------------- | --------------- | -| X64 | ● | ● | | -| MIPS64 | | | ● | -| ARM64 | | ○ | ○ | -| Alpha64 | | | ○ | +| | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **Other Linux** | **UOS** | **Kylin** | **Ningsi V60/V80** | **HUAWEI EulerOS** | +| ------------------ | ----------------- | ---------------- | ---------------- | --------------- | ------- | --------- | ------------------ | ------------------ | +| X64 | ● | ● | ● | | ● | ● | ● | | +| Raspberry Pi ARM64 | | | | ● | | | | | +| HUAWEI cloud ARM64 | | | | | | | | ● | Note: ● means officially tested and verified, ○ means unofficially tested and verified. diff --git a/docs/en/14-reference/12-directory.md b/docs/en/14-reference/12-directory.md index d6cffd22e0..118bce8037 100644 --- a/docs/en/14-reference/12-directory.md +++ b/docs/en/14-reference/12-directory.md @@ -25,7 +25,6 @@ All executable files of TDengine are in the _/usr/local/taos/bin_ directory by d - _taosBenchmark_: TDengine testing tool - _remove.sh_: script to uninstall TDengine, please execute it carefully, link to the **rmtaos** command in the /usr/bin directory. Will remove the TDengine installation directory `/usr/local/taos`, but will keep `/etc/taos`, `/var/lib/taos`, `/var/log/taos` - _taosadapter_: server-side executable that provides RESTful services and accepts writing requests from a variety of other softwares -- _tarbitrator_: provides arbitration for two-node cluster deployments - _TDinsight.sh_: script to download TDinsight and install it - _set_core.sh_: script for setting up the system to generate core dump files for easy debugging - _taosd-dump-cfg.gdb_: script to facilitate debugging of taosd's gdb execution. diff --git a/docs/zh/14-reference/08-taos-shell.md b/docs/zh/14-reference/08-taos-shell.md index d4a516d2d2..2f3b551502 100644 --- a/docs/zh/14-reference/08-taos-shell.md +++ b/docs/zh/14-reference/08-taos-shell.md @@ -48,29 +48,30 @@ taos> SET MAX_BINARY_DISPLAY_WIDTH ; 您可通过配置命令行参数来改变 TDengine CLI 的行为。以下为常用的几个命令行参数： -- -h, --host=HOST: 要连接的 TDengine 服务端所在服务器的 FQDN, 默认为连接本地服务 -- -P, --port=PORT: 指定服务端所用端口号 -- -u, --user=USER: 连接时使用的用户名 -- -p, --password=PASSWORD: 连接服务端时使用的密码 +- -h HOST: 要连接的 TDengine 服务端所在服务器的 FQDN, 默认为连接本地服务 +- -P PORT: 指定服务端所用端口号 +- -u USER: 连接时使用的用户名 +- -p PASSWORD: 连接服务端时使用的密码 - -?, --help: 打印出所有命令行参数 还有更多其他参数： -- -c, --config-dir: 指定配置文件目录，Linux 环境下默认为 `/etc/taos`，该目录下的配置文件默认名称为 `taos.cfg` -- -C, --dump-config: 打印 -c 指定的目录中 `taos.cfg` 的配置参数 -- -d, --database=DATABASE: 指定连接到服务端时使用的数据库 -- -D, --directory=DIRECTORY: 导入指定路径中的 SQL 脚本文件 -- -f, --file=FILE: 以非交互模式执行 SQL 脚本文件。文件中一个 SQL 语句只能占一行 -- -k, --check=CHECK: 指定要检查的表 -- -l, --pktlen=PKTLEN: 网络测试时使用的测试包大小 -- -n, --netrole=NETROLE: 网络连接测试时的测试范围，默认为 `startup`, 可选值为 `client`、`server`、`rpc`、`startup`、`sync`、`speed` 和 `fqdn` 之一 -- -r, --raw-time: 将时间输出出无符号 64 位整数类型(即 C 语音中 uint64_t) -- -s, --commands=COMMAND: 以非交互模式执行的 SQL 命令 -- -S, --pkttype=PKTTYPE: 指定网络测试所用的包类型，默认为 TCP。只有 netrole 为 `speed` 时既可以指定为 TCP 也可以指定为 UDP -- -T, --thread=THREADNUM: 以多线程模式导入数据时的线程数 -- -s, --commands: 在不进入终端的情况下运行 TDengine 命令 -- -z, --timezone=TIMEZONE: 指定时区，默认为本地时区 -- -V, --version: 打印出当前版本号 +- -a AUTHSTR: 连接服务端的授权信息 +- -A: 通过用户名和密码计算授权信息 +- -c CONFIGDIR: 指定配置文件目录，Linux 环境下默认为 `/etc/taos`，该目录下的配置文件默认名称为 `taos.cfg` +- -C: 打印 -c 指定的目录中 `taos.cfg` 的配置参数 +- -d DATABASE: 指定连接到服务端时使用的数据库 +- -f FILE: 以非交互模式执行 SQL 脚本文件。文件中一个 SQL 语句只能占一行 +- -k: 测试服务端运行状态，0: unavailable，1: network ok，2: service ok，3: service degraded，4: exiting +- -l PKTLEN: 网络测试时使用的测试包大小 +- -n NETROLE: 网络连接测试时的测试范围，默认为 `client`, 可选值为 `client`、`server` +- -N PKTNUM: 网络测试时使用的测试包数量 +- -r: 将时间输出出无符号 64 位整数类型(即 C 语音中 uint64_t) +- -s COMMAND: 以非交互模式执行的 SQL 命令 +- -t: 测试服务端启动状态，状态同-k +- -w DISPLAYWIDTH: 客户端列显示宽度 +- -z TIMEZONE: 指定时区，默认为本地时区 +- -V: 打印出当前版本号 示例： diff --git a/docs/zh/14-reference/09-support-platform/index.md b/docs/zh/14-reference/09-support-platform/index.md index d396e4be1f..e676ab845a 100644 --- a/docs/zh/14-reference/09-support-platform/index.md +++ b/docs/zh/14-reference/09-support-platform/index.md @@ -5,18 +5,11 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" ## TDengine 服务端支持的平台列表 -| | **CentOS 7/8** | **Ubuntu 16/18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **华为 EulerOS** | -| ------------ | -------------- | ------------------- | --------------- | ------------ | ----------------- | ---------------- | ---------------- | -| X64 | ● | ● | | ○ | ● | ● | ● | -| 龙芯 MIPS64 | | | ● | | | | | -| 鲲鹏 ARM64 | | ○ | ○ | | ● | | | -| 申威 Alpha64 | | | ○ | ● | | | | -| 飞腾 ARM64 | | ○ 优麒麟 | | | | | | -| 海光 X64 | ● | ● | ● | ○ | ● | ● | | -| 瑞芯微 ARM64 | | | ○ | | | | | -| 全志 ARM64 | | | ○ | | | | | -| 炬力 ARM64 | | | ○ | | | | | -| 华为云 ARM64 | | | | | | | ● | +| | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **华为 EulerOS** | +| ------------ | ----------------- | ---------------- | ---------------- | --------------- | ------------ | ----------------- | ---------------- | ---------------- | +| X64 | ● | ● | ● | | ● | ● | ● | | +| 树莓派 ARM64 | | | | ● | | | | | +| 华为云 ARM64 | | | | | | | | ● | 注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。 diff --git a/docs/zh/14-reference/12-directory.md b/docs/zh/14-reference/12-directory.md index 0caf7e03c3..262eb99fa5 100644 --- a/docs/zh/14-reference/12-directory.md +++ b/docs/zh/14-reference/12-directory.md @@ -25,7 +25,6 @@ TDengine 的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 - _taosBenchmark_：TDengine 测试工具 - _remove.sh_：卸载 TDengine 的脚本，请谨慎执行，链接到/usr/bin 目录下的**rmtaos**命令。会删除 TDengine 的安装目录/usr/local/taos，但会保留/etc/taos、/var/lib/taos、/var/log/taos - _taosadapter_: 提供 RESTful 服务和接受其他多种软件写入请求的服务端可执行文件 -- _tarbitrator_: 提供双节点集群部署的仲裁功能 - _TDinsight.sh_：用于下载 TDinsight 并安装的脚本 - _set_core.sh_：用于方便调试设置系统生成 core dump 文件的脚本 - _taosd-dump-cfg.gdb_：用于方便调试 taosd 的 gdb 执行脚本。 From d3b2bcb61227e4665404a3286cd9f75c60947508 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 19:39:01 +0800 Subject: [PATCH 85/96] doc: change 3.0 doc --- tools/shell/src/shellArguments.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 88ef46c5d6..818cc5425b 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -38,7 +38,7 @@ #define SHELL_STARTUP "Check the details of the service status." #define SHELL_WIDTH "Set the default binary display width, default is 30." #define SHELL_NET_ROLE "Net role when network connectivity test, options: client|server." -#define SHELL_PKG_LEN "Packet length used for net test, default is 1024 bytes." +#define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes." #define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." #define SHELL_VERSION "Print program version." #define SHELL_EMAIL "" @@ -62,7 +62,7 @@ void shellPrintHelp() { printf("%s%s%s%s\r\n", indent, "-f,", indent, SHELL_FILE); printf("%s%s%s%s\r\n", indent, "-h,", indent, SHELL_HOST); printf("%s%s%s%s\r\n", indent, "-k,", indent, SHELL_CHECK); - printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKG_LEN); + printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN); printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE); printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM); printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD); @@ -105,11 +105,11 @@ static struct argp_option shellOptions[] = { {"startup", 't', 0, 0, SHELL_STARTUP}, {"display-width", 'w', "WIDTH", 0, SHELL_WIDTH}, {"netrole", 'n', "NETROLE", 0, SHELL_NET_ROLE}, - {"pktlen", 'l', "PKTLEN", 0, SHELL_PKG_LEN}, + {"pktlen", 'l', "PKTLEN", 0, SHELL_PKT_LEN}, #ifdef WEBSOCKET {"dsn", 'E', "DSN", 0, SHELL_DSN}, {"restful", 'R', 0, 0, SHELL_REST}, - {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, + {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, {0}, @@ -228,7 +228,7 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { SShellArgs *pArgs = &shell.args; for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0) { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) { shellParseSingleOpt('?', NULL); return 0; } From ebf8755d9da1294ba1280295643d99903139aeea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 19:58:23 +0800 Subject: [PATCH 86/96] fix(query): remove invalid sma filter. --- source/libs/executor/src/scanoperator.c | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a211542de1..e59125a9bc 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -319,6 +319,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); return TSDB_CODE_SUCCESS; } else { + qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); *status = FUNC_DATA_REQUIRED_DATA_LOAD; } } @@ -326,20 +327,19 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD); // try to filter data block according to sma info - if (pTableScanInfo->pFilterNode != NULL) { - if (!loadSMA) { - doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); - } + if (pTableScanInfo->pFilterNode != NULL && (!loadSMA)) { + bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); + if (success) { + size_t size = taosArrayGetSize(pBlock->pDataBlock); + bool keep = doFilterByBlockSMA(pTableScanInfo->pFilterNode, pBlock->pBlockAgg, size, pBlockInfo->rows); + if (!keep) { + qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + pCost->filterOutBlocks += 1; + (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - bool keep = doFilterByBlockSMA(pTableScanInfo->pFilterNode, pBlock->pBlockAgg, taosArrayGetSize(pBlock->pDataBlock), - pBlockInfo->rows); - if (!keep) { - qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->filterOutBlocks += 1; - (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; + } } } From d8455841e01645750def9fff5bff6166233ff626 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 20:12:32 +0800 Subject: [PATCH 87/96] refactor: do some internal refactor. --- include/libs/function/function.h | 16 +- source/libs/executor/src/scanoperator.c | 75 - .../inc/{taggfunction.h => tfunctionInt.h} | 17 +- source/libs/function/src/builtinsimpl.c | 10 +- source/libs/function/src/taggfunction.c | 3910 ----------------- source/libs/function/src/texpr.c | 67 - source/libs/function/src/tfunctionInt.c | 86 + 7 files changed, 93 insertions(+), 4088 deletions(-) rename source/libs/function/inc/{taggfunction.h => tfunctionInt.h} (76%) delete mode 100644 source/libs/function/src/taggfunction.c delete mode 100644 source/libs/function/src/texpr.c create mode 100644 source/libs/function/src/tfunctionInt.c diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 72732ee198..e708a2c42d 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -54,10 +54,6 @@ typedef struct SFuncExecFuncs { FExecCombine combine; } SFuncExecFuncs; -typedef struct SFileBlockInfo { - int32_t numBlocksOfStep; -} SFileBlockInfo; - #define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results #define TOP_BOTTOM_QUERY_LIMIT 100 @@ -171,8 +167,6 @@ typedef struct tExprNode { }; } tExprNode; -void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); - struct SScalarParam { bool colAlloced; SColumnInfoData *columnData; @@ -182,14 +176,10 @@ struct SScalarParam { int32_t numOfRows; }; -int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, - bool isSuperTable); - -void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num); -void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); +void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock); -bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); -bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); +bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); +bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); typedef struct SPoint { int64_t key; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e59125a9bc..01a99fabee 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2458,81 +2458,6 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; -#if 0 - int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; - - STagScanInfo *pInfo = pOperator->info; - SSDataBlock *pRes = pInfo->pRes; - - int32_t count = 0; - SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); - - int32_t functionId = getExprFunctionId(&pOperator->exprSupp.pExprInfo[0]); - if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id - assert(pQueryAttr->numOfOutput == 1); - - SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; - int32_t rsize = pExprInfo->base.resSchema.bytes; - - count = 0; - - int16_t bytes = pExprInfo->base.resSchema.bytes; - int16_t type = pExprInfo->base.resSchema.type; - - for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) { - bytes = pQueryAttr->tagColList[i].bytes; - type = pQueryAttr->tagColList[i].type; - break; - } - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - STableQueryInfo *item = taosArrayGetP(pa, i); - - char *output = pColInfo->pData + count * rsize; - varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); - - output = varDataVal(output); - STableId* id = TSDB_TABLEID(item->pTable); - - *(int16_t *)output = 0; - output += sizeof(int16_t); - - *(int64_t *)output = id->uid; // memory align problem, todo serialize - output += sizeof(id->uid); - - *(int32_t *)output = id->tid; - output += sizeof(id->tid); - - *(int32_t *)output = pQueryAttr->vgId; - output += sizeof(pQueryAttr->vgId); - - char* data = NULL; - if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes); - } - - doSetTagValueToResultBuf(output, data, type, bytes); - count += 1; - } - - //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); - } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - *(int64_t*)pColInfo->pData = pInfo->totalTables; - count = 1; - - pOperator->status = OP_EXEC_DONE; - //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); - } else { // return only the tags|table name etc. -#endif - STagScanInfo* pInfo = pOperator->info; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; SSDataBlock* pRes = pInfo->pRes; diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/tfunctionInt.h similarity index 76% rename from source/libs/function/inc/taggfunction.h rename to source/libs/function/inc/tfunctionInt.h index 669c2635b5..8f6cbc977e 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/tfunctionInt.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TAGGFUNCTION_H -#define TDENGINE_TAGGFUNCTION_H +#ifndef TDENGINE_TFUNCTIONINT_H +#define TDENGINE_TFUNCTIONINT_H #ifdef __cplusplus extern "C" { @@ -28,17 +28,6 @@ extern "C" { #include "function.h" #include "tudf.h" -#define AVG_FUNCTION_INTER_BUFFER_SIZE 50 - -#define DATA_SET_FLAG ',' // to denote the output area has data, not null value -#define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) - -typedef struct SInterpInfoDetail { - TSKEY ts; // interp specified timestamp - int8_t type; - int8_t primaryCol; -} SInterpInfoDetail; - bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval); /** @@ -57,4 +46,4 @@ static FORCE_INLINE void initResultRowEntry(SResultRowEntryInfo *pResInfo, int32 } #endif -#endif // TDENGINE_TAGGFUNCTION_H +#endif // TDENGINE_TFUNCTIONINT_H diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index eb5d05c540..3e6e534328 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -18,10 +18,10 @@ #include "function.h" #include "query.h" #include "querynodes.h" -#include "taggfunction.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" +#include "tfunctionInt.h" #include "tglobal.h" #include "thistogram.h" #include "tpercentile.h" @@ -312,14 +312,6 @@ typedef struct SGroupKeyInfo { #define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) #define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) -#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SqlFunctionCtx* __ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - __ctx->fpSet.process(__ctx); \ - } \ - } while (0); - #define DO_UPDATE_SUBSID_RES(ctx, ts) \ do { \ for (int32_t _i = 0; _i < (ctx)->subsidiaries.num; ++_i) { \ diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c deleted file mode 100644 index c8998b9d94..0000000000 --- a/source/libs/function/src/taggfunction.c +++ /dev/null @@ -1,3910 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "taosdef.h" -#include "tmsg.h" -#include "thash.h" -#include "ttypes.h" - -#include "function.h" -#include "taggfunction.h" -#include "tbuffer.h" -#include "tcompression.h" -#include "thistogram.h" -#include "tpercentile.h" -#include "ttszip.h" -#include "tdatablock.h" -#include "tudf.h" - -#define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) -#define GET_INPUT_DATA(x, y) ((char*) colDataGetData((x)->pInput, (y))) - -#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) -#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) - -#define GET_TRUE_DATA_TYPE() \ - int32_t type = 0; \ - if (pCtx->scanFlag == MERGE_STAGE) { \ - type = pCtx->resDataInfo.type; \ - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \ - } else { \ - type = pCtx->inputType; \ - } - -#define SET_VAL(ctx, numOfElem, res) \ - do { \ - if ((numOfElem) <= 0) { \ - break; \ - } \ - GET_RES_INFO(ctx)->numOfRes = (res); \ - } while (0) - -#define INC_INIT_VAL(ctx, res) (GET_RES_INFO(ctx)->numOfRes += (res)); - -#define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ - __ctx->tag.i = (ts); \ - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ - } \ - aggFunc[FUNCTION_TAG].addInput(__ctx); \ - } \ - } while (0) - -#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - aggFunc[FUNCTION_TAG].addInput(__ctx); \ - } \ - } while (0); - -void noop1(SqlFunctionCtx *UNUSED_PARAM(pCtx)) {} - -void doFinalizer(SqlFunctionCtx *pCtx) { cleanupResultRowEntry(GET_RES_INFO(pCtx)); } - -typedef struct tValuePair { - SVariant v; - int64_t timestamp; - char * pTags; // the corresponding tags of each record in the final result -} tValuePair; - -typedef struct SSpreadInfo { - double min; - double max; - int8_t hasResult; -} SSpreadInfo; - -typedef struct SSumInfo { - union { - int64_t isum; - uint64_t usum; - double dsum; - }; - int8_t hasResult; -} SSumInfo; - -// the attribute of hasResult is not needed since the num attribute would server as this purpose -typedef struct SAvgInfo { - double sum; - int64_t num; -} SAvgInfo; - -typedef struct SStddevInfo { - double avg; - int64_t num; - double res; - int8_t stage; -} SStddevInfo; - -typedef struct SStddevdstInfo { - int64_t num; - double res; -} SStddevdstInfo; - -typedef struct SFirstLastInfo { - int8_t hasResult; - TSKEY ts; -} SFirstLastInfo; - -typedef struct SFirstLastInfo SLastrowInfo; -typedef struct SPercentileInfo { - tMemBucket *pMemBucket; - int32_t stage; - double minval; - double maxval; - int64_t numOfElems; -} SPercentileInfo; - -typedef struct STopBotInfo { - int32_t num; - tValuePair **res; -} STopBotInfo; - -// leastsquares do not apply to super table -typedef struct SLeastsquaresInfo { - double mat[2][3]; - double startVal; - int64_t num; -} SLeastsquaresInfo; - -typedef struct SAPercentileInfo { - SHistogramInfo *pHisto; -} SAPercentileInfo; - -typedef struct STSCompInfo { - STSBuf *pTSBuf; -} STSCompInfo; - -typedef struct SRateInfo { - double correctionValue; - double firstValue; - TSKEY firstKey; - double lastValue; - TSKEY lastKey; - int8_t hasResult; // flag to denote has value - bool isIRate; // true for IRate functions, false for Rate functions -} SRateInfo; - -//typedef struct SDerivInfo { -// double prevValue; // previous value -// TSKEY prevTs; // previous timestamp -// bool ignoreNegative;// ignore the negative value -// int64_t tsWindow; // time window for derivative -// bool valueSet; // the value has been set already -//} SDerivInfo; - -typedef struct SResPair { - TSKEY key; - double avg; -} SResPair; - -void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { - pCell->initialized = false; -} - -int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock) { - int32_t maxRows = 0; - - for (int32_t j = 0; j < num; ++j) { -#if 0 - int32_t id = pCtx[j].functionId; - - /* - * ts, tag, tagprj function can not decide the output number of current query - * the number of output result is decided by main output - */ - if (id == FUNCTION_TS || id == FUNCTION_TAG || id == FUNCTION_TAGPRJ) { - continue; - } -#endif - SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); - if (pResInfo != NULL && maxRows < pResInfo->numOfRes) { - maxRows = pResInfo->numOfRes; - } - } - - assert(maxRows >= 0); - - blockDataEnsureCapacity(pResBlock, maxRows); - for(int32_t i = 0; i < num; ++i) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[i]); - if (pResInfo->numOfRes == 0) { - for(int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataAppend(pCol, j, NULL, true); // TODO add set null data api - } - } else { - for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataAppend(pCol, j, GET_ROWCELL_INTERBUF(pResInfo), false); - } - } - } - - pResBlock->info.rows = maxRows; - return maxRows; -} - -void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num) { - for (int32_t j = 0; j < num; ++j) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); - pResInfo->numOfRes = 0; - } -} - -bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { - assert(pEntry != NULL); - return pEntry->complete; -} - -bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry) { - return pEntry->initialized; -} -#if 0 -int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, - bool isSuperTable/*, SUdfInfo* pUdfInfo*/) { - if (!isValidDataType(dataType)) { -// qError("Illegal data type %d or data type length %d", dataType, dataBytes); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - - if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG_DUMMY || - functionId == FUNCTION_DIFF || functionId == FUNCTION_PRJ || functionId == FUNCTION_TAGPRJ || - functionId == FUNCTION_TAG || functionId == FUNCTION_INTERP) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - - if (functionId == FUNCTION_INTERP) { - pInfo->interBufSize = sizeof(SInterpInfoDetail); - } else { - pInfo->interBufSize = 0; - } - - return TSDB_CODE_SUCCESS; - } - - // (uid, tid) + VGID + TAGSIZE + VARSTR_HEADER_SIZE - if (functionId == FUNCTION_TID_TAG) { // todo use struct - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_BLKINFO) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = 16384; - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_COUNT) { - pInfo->type = TSDB_DATA_TYPE_BIGINT; - pInfo->bytes = sizeof(int64_t); - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_ARITHM) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_TS_COMP) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = 1; // this results is compressed ts data, only one byte - pInfo->interBufSize = POINTER_BYTES; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_DERIVATIVE) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); // this results is compressed ts data, only one byte - pInfo->interBufSize = sizeof(SDerivInfo); - return TSDB_CODE_SUCCESS; - } - - if (isSuperTable) { -// if (functionId < 0) { -// if (pUdfInfo->bufSize > 0) { -// pInfo->type = TSDB_DATA_TYPE_BINARY; -// pInfo->bytes = pUdfInfo->bufSize; -// pInfo->interBufSize = pInfo->bytes; -// } else { -// pInfo->type = pUdfInfo->resType; -// pInfo->bytes = pUdfInfo->resBytes; -// pInfo->interBufSize = pInfo->bytes; -// } -// -// return TSDB_CODE_SUCCESS; -// } - - if (functionId == FUNCTION_MIN || functionId == FUNCTION_MAX) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(dataBytes + DATA_SET_FLAG_SIZE); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_SUM) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SSumInfo); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_AVG) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SAvgInfo); - pInfo->interBufSize = pInfo->bytes; - return TSDB_CODE_SUCCESS; - - } else if (functionId >= FUNCTION_RATE && functionId <= FUNCTION_IRATE) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(SRateInfo); - pInfo->interBufSize = sizeof(SRateInfo); - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_SPREAD) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SSpreadInfo); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_APERCT) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo) + sizeof(SAPercentileInfo); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_LAST_ROW) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(sizeof(SLastrowInfo) + dataBytes); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_TWA) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(STwaInfo); - pInfo->interBufSize = pInfo->bytes; - return TSDB_CODE_SUCCESS; - } - } - - if (functionId == FUNCTION_SUM) { - if (IS_SIGNED_NUMERIC_TYPE(dataType)) { - pInfo->type = TSDB_DATA_TYPE_BIGINT; - } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { - pInfo->type = TSDB_DATA_TYPE_UBIGINT; - } else { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - } - - pInfo->bytes = sizeof(int64_t); - pInfo->interBufSize = sizeof(SSumInfo); - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_APERCT) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = - sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_TWA) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(STwaInfo); - return TSDB_CODE_SUCCESS; - } - -// if (functionId < 0) { -// pInfo->type = pUdfInfo->resType; -// pInfo->bytes = pUdfInfo->resBytes; -// -// if (pUdfInfo->bufSize > 0) { -// pInfo->interBufSize = pUdfInfo->bufSize; -// } else { -// pInfo->interBufSize = pInfo->bytes; -// } -// -// return TSDB_CODE_SUCCESS; -// } - - if (functionId == FUNCTION_AVG) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SAvgInfo); - } else if (functionId >= FUNCTION_RATE && functionId <= FUNCTION_IRATE) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SRateInfo); - } else if (functionId == FUNCTION_STDDEV) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SStddevInfo); - } else if (functionId == FUNCTION_MIN || functionId == FUNCTION_MAX) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - pInfo->interBufSize = dataBytes + DATA_SET_FLAG_SIZE; - } else if (functionId == FUNCTION_FIRST || functionId == FUNCTION_LAST) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - pInfo->interBufSize = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); - } else if (functionId == FUNCTION_SPREAD) { - pInfo->type = (int16_t)TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SSpreadInfo); - } else if (functionId == FUNCTION_PERCT) { - pInfo->type = (int16_t)TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = (int16_t)sizeof(double); - pInfo->interBufSize = (int16_t)sizeof(SPercentileInfo); - } else if (functionId == FUNCTION_LEASTSQR) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = TMAX(AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string - pInfo->interBufSize = pInfo->bytes; - } else if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); - pInfo->interBufSize = pInfo->bytes; - } else if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - - size_t size = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; - - // the output column may be larger than sizeof(STopBotInfo) - pInfo->interBufSize = (int32_t)size; - } else if (functionId == FUNCTION_LAST_ROW) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - pInfo->interBufSize = dataBytes; - } else if (functionId == FUNCTION_STDDEV_DST) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SStddevdstInfo); - pInfo->interBufSize = (pInfo->bytes); - - } else { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} -#endif - -static bool function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (pResultInfo->initialized) { - return false; - } - - memset(pCtx->pOutput, 0, (size_t)pCtx->resDataInfo.bytes); - initResultRowEntry(pResultInfo, pCtx->resDataInfo.interBufSize); - return true; -} -#if 0 -/** - * in handling the stable query, function_finalizer is called after the secondary - * merge being completed, during the first merge procedure, which is executed at the - * vnode side, the finalize will never be called. - * - * @param pCtx - */ -static void function_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); -// if (pResInfo->hasResult != DATA_SET_FLAG) { // TODO set the correct null value -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// } - - doFinalizer(pCtx); -} - -/** - * 1. If the column value for filter exists, we need to load the SFields, which serves - * as the pre-filter to decide if the actual data block is required or not. - * 2. If it queries on the non-primary timestamp column, SFields is also required to get the not-null value. - * - * @param colId - * @param filterCols - * @return - */ -int32_t countRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - return BLK_DATA_NOT_LOAD; - } else { - return BLK_DATA_SMA_LOAD; - } -} - -int32_t noDataRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_NOT_LOAD; -} -#define LIST_ADD_N_DOUBLE_FLOAT(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - SET_DOUBLE_VAL(&(x) , GET_DOUBLE_VAL(&(x)) + GET_FLOAT_VAL(&(d)[i])); \ - (numOfElem)++; \ - } \ - } while(0) -#define LIST_ADD_N_DOUBLE(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - SET_DOUBLE_VAL(&(x) , (x) + (d)[i]); \ - (numOfElem)++; \ - } \ - } while(0) - -#define LIST_ADD_N(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - (x) += (d)[i]; \ - (numOfElem)++; \ - } \ - } while(0) - -#define UPDATE_DATA(ctx, left, right, num, sign, k) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ - DO_UPDATE_TAG_COLUMNS(ctx, k); \ - (num) += 1; \ - } \ - } while (0) - -#define DUPATE_DATA_WITHOUT_TS(ctx, left, right, num, sign) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ - DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx); \ - (num) += 1; \ - } \ - } while (0) - -#define LOOPCHECK_N(val, list, ctx, tsdbType, sign, num) \ - for (int32_t i = 0; i < ((ctx)->size); ++i) { \ - if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ - continue; \ - } \ - TSKEY key = (ctx)->ptsList != NULL? GET_TS_DATA(ctx, i):0; \ - UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \ - } - -#define TYPED_LOOPCHECK_N(type, data, list, ctx, tsdbType, sign, notNullElems) \ - do { \ - type *_data = (type *)data; \ - type *_list = (type *)list; \ - LOOPCHECK_N(*_data, _list, ctx, tsdbType, sign, notNullElems); \ - } while (0) - -static int32_t statisRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_SMA_LOAD; -} - -static int32_t dataBlockRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_DATA_LOAD; -} - -// todo: if column in current data block are null, opt for this case -static int32_t firstFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order == TSDB_ORDER_DESC) { - return BLK_DATA_NOT_LOAD; - } - - // no result for first query, data block is required - if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { - return BLK_DATA_DATA_LOAD; - } else { - return BLK_DATA_NOT_LOAD; - } -} - -static int32_t lastFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { -// if (pCtx->order != pCtx->param[0].param.i) { -// return BLK_DATA_NOT_LOAD; -// } - - if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { - return BLK_DATA_DATA_LOAD; - } else { - return BLK_DATA_NOT_LOAD; - } -} - -static int32_t firstDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order == TSDB_ORDER_DESC) { - return BLK_DATA_NOT_LOAD; - } - - // not initialized yet, it is the first block, load it. - if (pCtx->pOutput == NULL) { - return BLK_DATA_DATA_LOAD; - } - - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is - // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG) { - return BLK_DATA_DATA_LOAD; - } else { // data in current block is not earlier than current result - return (pInfo->ts <= w->skey) ? BLK_DATA_NOT_LOAD : BLK_DATA_DATA_LOAD; - } -} - -static int32_t lastDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { -// if (pCtx->order != pCtx->param[0].param.i) { -// return BLK_DATA_NOT_LOAD; -// } - - // not initialized yet, it is the first block, load it. - if (pCtx->pOutput == NULL) { - return BLK_DATA_DATA_LOAD; - } - - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is - // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG) { - return BLK_DATA_DATA_LOAD; - } else { - return (pInfo->ts > w->ekey) ? BLK_DATA_NOT_LOAD : BLK_DATA_DATA_LOAD; - } -} - -////////////////////////////////////////////////////////////////////////////////////////////// -/* - * The intermediate result of average is kept in the interResultBuf. - * For super table query, once the avg_function/avg_function_f is finished, copy the intermediate - * result into output buffer. - */ -static void avg_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - // NOTE: keep the intermediate result into the interResultBuf - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - double *pVal = &pAvgInfo->sum; - - if (pCtx->isAggSet) { // Pre-aggregation - notNullElems = pCtx->size - pCtx->agg.numOfNull; - assert(notNullElems >= 0); - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += pCtx->agg.sum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += (uint64_t) pCtx->agg.sum; - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->agg.sum)); - } - } else { - void *pData = GET_INPUT_DATA_LIST(pCtx); - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(*pVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(*pVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(*pVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_ADD_N_DOUBLE(*pVal, pCtx, pData, double, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_ADD_N_DOUBLE_FLOAT(*pVal, pCtx, pData, float, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); - } - } - - if (!pCtx->hasNull) { - assert(notNullElems == pCtx->size); - } - - SET_VAL(pCtx, notNullElems, 1); - pAvgInfo->num += notNullElems; - - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); - } -} - -static void avg_func_merge(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - double *sum = (double*) pCtx->pOutput; - char *input = GET_INPUT_DATA_LIST(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SAvgInfo *pInput = (SAvgInfo *)input; - if (pInput->num == 0) { // current input is null - continue; - } - - SET_DOUBLE_VAL(sum, *sum + pInput->sum); - - // keep the number of data into the temp buffer - *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num; - } -} - -/* - * the average value is calculated in finalize routine, since current routine does not know the exact number of points - */ -static void avg_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->scanFlag == MERGE_STAGE) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput,(*(double *)pCtx->pOutput) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo)); - } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY - assert(IS_NUMERIC_TYPE(pCtx->inputType)); - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pAvgInfo->num == 0) { // all data are NULL or empty table - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pAvgInfo->sum / pAvgInfo->num); - } - - // cannot set the numOfIteratedElems again since it is set during previous iteration - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////////////////// - -static bool min_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; // not initialized since it has been initialized - } - - GET_TRUE_DATA_TYPE(); - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->pOutput) = INT8_MAX; - break; - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *) pCtx->pOutput = UINT8_MAX; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->pOutput) = INT16_MAX; - break; - case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->pOutput) = UINT16_MAX; - break; - case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->pOutput) = INT32_MAX; - break; - case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->pOutput) = UINT32_MAX; - break; - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->pOutput) = INT64_MAX; - break; - case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->pOutput) = UINT64_MAX; - break; - case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->pOutput) = FLT_MAX; - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)pCtx->pOutput), DBL_MAX); - break; - default: - assert(0); -// qError("illegal data type:%d in min/max query", pCtx->inputType); - } - - return true; -} - -static bool max_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; // not initialized since it has been initialized - } - - GET_TRUE_DATA_TYPE(); - - switch (type) { - case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->pOutput) = INT32_MIN; - break; - case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->pOutput) = -FLT_MAX; - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)pCtx->pOutput), -DBL_MAX); - break; - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->pOutput) = INT64_MIN; - break; - case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->pOutput) = INT16_MIN; - break; - case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->pOutput) = INT8_MIN; - break; - case TSDB_DATA_TYPE_UTINYINT: - *((uint8_t *)pCtx->pOutput) = 0; - break; - default: - assert(0); -// qError("illegal data type:%d in min/max query", pCtx->inputType); - } - - return true; -} - -/* - * the output result of min/max function is the final output buffer, not the intermediate result buffer - */ -static int32_t minmax_merge_impl(SqlFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { - int32_t notNullElems = 0; -#if 0 - GET_TRUE_DATA_TYPE(); - assert(pCtx->stableQuery); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *input = GET_INPUT_DATA(pCtx, i); - if (input[bytes] != DATA_SET_FLAG) { - continue; - } - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t v = GET_INT8_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int8_t *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t v = GET_INT16_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int16_t *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t v = GET_INT32_VAL(input); - if ((*(int32_t *)output < v) ^ isMin) { - *(int32_t *)output = v; - - for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - SqlFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; - aggFunc[FUNCTION_TAG].addInput(__ctx); - } - - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float v = GET_FLOAT_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(float *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double v = GET_DOUBLE_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(double *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t v = GET_INT64_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int64_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t v = GET_UINT8_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint8_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t v = GET_UINT16_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint16_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UINT: { - uint32_t v = GET_UINT32_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint32_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t v = GET_UINT64_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint64_t *)output, v, notNullElems, isMin); - break; - } - - default: - break; - } - } -#endif - - return notNullElems; -} - -static void min_func_merge(SqlFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 1); - - SET_VAL(pCtx, notNullElems, 1); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void max_func_merge(SqlFunctionCtx *pCtx) { - int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 0); - - SET_VAL(pCtx, numOfElem, 1); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - if (numOfElem > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -#define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, _type, num) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], (_type))) { \ - continue; \ - } \ - (num) += 1; \ - (r) += TPOW2(((type *)d)[i] - (delta)); \ - } - -static void stddev_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->scanFlag == REPEAT_SCAN && pStd->stage == 0) { - pStd->stage++; - avg_finalizer(pCtx); - - pResInfo->initialized = true; // set it initialized to avoid re-initialization - - // save average value into tmpBuf, for second stage scan - SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); - - pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); - assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); - } - - if (pStd->stage == 0) { - // the first stage is to calculate average value - avg_function(pCtx); - } else if (pStd->num > 0) { - // the second stage to calculate standard deviation - // if pStd->num == 0, there are no numbers in the first round check. No need to do the second round - double *retVal = &pStd->res; - double avg = pStd->avg; - - void *pData = GET_INPUT_DATA_LIST(pCtx); - int32_t num = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { - continue; - } - num += 1; - *retVal += TPOW2(((int32_t *)pData)[i] - avg); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - LOOP_STDDEV_IMPL(uint8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UINT: { - LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - default: - assert(0); -// qError("stddev function not support data type:%d", pCtx->inputType); - } - - SET_VAL(pCtx, 1, 1); - } -} - -static void stddev_finalizer(SqlFunctionCtx *pCtx) { - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - if (pStd->num <= 0) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else { - double *retValue = (double *)pCtx->pOutput; - SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num)); - SET_VAL(pCtx, 1, 1); - } - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////// -int32_t tsCompare(const void* p1, const void* p2) { - TSKEY k = *(TSKEY*)p1; - SResPair* pair = (SResPair*)p2; - - if (k == pair->key) { - return 0; - } else { - return k < pair->key? -1:1; - } -} - -////////////////////////////////////////////////////////////////////////////////////// -static bool first_last_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - // used to keep the timestamp for comparison -// pCtx->param[1].param.nType = 0; -// pCtx->param[1].param.i = 0; - - return true; -} - -// todo opt for null block -static void first_function(SqlFunctionCtx *pCtx) { - if (pCtx->order == TSDB_ORDER_DESC) { - return; - } - - int32_t notNullElems = 0; - - // handle the null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - if (pCtx->ptsList != NULL) { - TSKEY k = GET_TS_DATA(pCtx, i); -// DO_UPDATE_TAG_COLUMNS(pCtx, k); - } - - SResultRowEntryInfo *pInfo = GET_RES_INFO(pCtx); -// pInfo->hasResult = DATA_SET_FLAG; - pInfo->complete = true; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void first_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = GET_TS_LIST(pCtx); - - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - - if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) { - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); - } -} - -/* - * format of intermediate result: "timestamp,value" need to compare the timestamp in the first part (before the comma) - * to decide if the value is earlier than current intermediate result - */ -static void first_dist_function(SqlFunctionCtx *pCtx) { - /* - * do not to check data in the following cases: - * 1. data block that are not loaded - * 2. scan data files in desc order - */ - if (pCtx->order == TSDB_ORDER_DESC) { - return; - } - - int32_t notNullElems = 0; - - // find the first not null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - first_data_assign_impl(pCtx, data, i); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void first_dist_func_merge(SqlFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - char * pData = GET_INPUT_DATA_LIST(pCtx); - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->resDataInfo.bytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - // The param[1] is used to keep the initial value of max ts value -// if (pCtx->param[1].param.nType != pCtx->resDataInfo.type || pCtx->param[1].param.i > pInput->ts) { -// memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); -// pCtx->param[1].param.i = pInput->ts; -// pCtx->param[1].param.nType = pCtx->resDataInfo.type; -// -//// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); -// } - - SET_VAL(pCtx, 1, 1); -// GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -////////////////////////////////////////////////////////////////////////////////////////// -/* - * last function: - * 1. since the last block may be all null value, so, we simply access the last block is not valid - * each block need to be checked. - * 2. If numOfNull == pBlock->numOfBlocks, the whole block is empty. Otherwise, there is at - * least one data in this block that is not null.(TODO opt for this case) - */ -static void last_function(SqlFunctionCtx *pCtx) { -// if (pCtx->order != pCtx->param[0].param.i) { -// return; -// } - - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - if (pCtx->order == TSDB_ORDER_DESC) { - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { - continue; - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - - TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); - - //pResInfo->hasResult = DATA_SET_FLAG; - pResInfo->complete = true; // set query completed on this column - notNullElems++; - break; - } - } else { // ascending order - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { - continue; - } - - TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; - - char* buf = GET_ROWCELL_INTERBUF(pResInfo); -// if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { -// //pResInfo->hasResult = DATA_SET_FLAG; -// memcpy(pCtx->pOutput, data, pCtx->inputBytes); -// -// *(TSKEY*)buf = ts; -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); -// } - - notNullElems++; - break; - } - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void last_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = GET_TS_LIST(pCtx); - - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - - if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) { -#if defined(_DEBUG_VIEW) - qDebug("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData); -#endif - - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); - } -} - -static void last_dist_function(SqlFunctionCtx *pCtx) { - /* - * 1. for scan data is not the required order - * 2. for data blocks that are not loaded, no need to check data - */ -// if (pCtx->order != pCtx->param[0].param.i) { -// return; -// } - - int32_t notNullElems = 0; - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - if (!pCtx->requireNull) { - continue; - } - } - - last_data_assign_impl(pCtx, data, i); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -/* - * in the secondary merge(local reduce), the output is limited by the - * final output size, so the main difference between last_dist_func_merge and second_merge - * is: the output data format in computing - */ -static void last_dist_func_merge(SqlFunctionCtx *pCtx) { - char *pData = GET_INPUT_DATA_LIST(pCtx); - - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->resDataInfo.bytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - /* - * param[1] used to keep the corresponding timestamp to decide if current result is - * the true last result - */ - if (pCtx->param[1].param.nType != pCtx->resDataInfo.type || pCtx->param[1].param.i < pInput->ts) { - memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); - pCtx->param[1].param.i = pInput->ts; - pCtx->param[1].param.nType = pCtx->resDataInfo.type; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } - - SET_VAL(pCtx, 1, 1); -// GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -////////////////////////////////////////////////////////////////////////////////// -/* - * NOTE: last_row does not use the interResultBuf to keep the result - */ -static void last_row_function(SqlFunctionCtx *pCtx) { - assert(pCtx->size >= 1); - char *pData = GET_INPUT_DATA_LIST(pCtx); - - // assign the last element in current data block - assignVal(pCtx->pOutput, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - // set the result to final result buffer in case of super table query - if (pCtx->stableQuery) { - SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->pOutput + pCtx->inputBytes); - pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); - pInfo1->hasResult = DATA_SET_FLAG; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); - } else { - TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1); -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); - } - - SET_VAL(pCtx, pCtx->size, 1); -} - -static void last_row_finalizer(SqlFunctionCtx *pCtx) { - // do nothing at the first stage - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); -// if (pResInfo->hasResult != DATA_SET_FLAG) { -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// return; -// } - - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////// -static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int64_t tsKey, char *pTags, - SSubsidiaryResInfo *pTagInfo, int16_t stage) { - dst->v.nType = type; - dst->v.i = *(int64_t *)val; - dst->timestamp = tsKey; - - int32_t size = 0; - if (stage == MERGE_STAGE) { -// memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); - } else { // the tags are dumped from the ctx tag fields -// for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { -// SqlFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; -// if (ctx->functionId == FUNCTION_TS_DUMMY) { -// ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; -// ctx->tag.i = tsKey; -// } -// -// taosVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); -// size += pTagInfo->pTagCtxList[i]->resDataInfo.bytes; -// } - } -} - -#define VALUEPAIRASSIGN(dst, src, __l) \ - do { \ - (dst)->timestamp = (src)->timestamp; \ - (dst)->v = (src)->v; \ - memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ - } while (0) - -static int32_t topBotComparFn(const void *p1, const void *p2, const void *param) -{ - uint16_t type = *(uint16_t *) param; - tValuePair *val1 = *(tValuePair **) p1; - tValuePair *val2 = *(tValuePair **) p2; - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - if (val1->v.i == val2->v.i) { - return 0; - } - - return (val1->v.i > val2->v.i) ? 1 : -1; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - if (val1->v.u == val2->v.u) { - return 0; - } - - return (val1->v.u > val2->v.u) ? 1 : -1; - } - - if (val1->v.d == val2->v.d) { - return 0; - } - - return (val1->v.d > val2->v.d) ? 1 : -1; -} - -static void topBotSwapFn(void *dst, void *src, const void *param) -{ - char tag[32768]; - tValuePair temp; - uint16_t tagLen = *(uint16_t *) param; - tValuePair *vdst = *(tValuePair **) dst; - tValuePair *vsrc = *(tValuePair **) src; - - memset(tag, 0, sizeof(tag)); - temp.pTags = tag; - - VALUEPAIRASSIGN(&temp, vdst, tagLen); - VALUEPAIRASSIGN(vdst, vsrc, tagLen); - VALUEPAIRASSIGN(vsrc, &temp, tagLen); -} - -static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SSubsidiaryResInfo *pTagInfo, char *pTags, int16_t stage) { - SVariant val = {0}; - taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); - - tValuePair **pList = pInfo->res; - assert(pList != NULL); - - if (pInfo->num < maxLen) { - valuePairAssign(pList[pInfo->num], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - -// taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); - - pInfo->num++; - } else { - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pList[0]->v.i) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pList[0]->v.u) || - (IS_FLOAT_TYPE(type) && val.d > pList[0]->v.d)) { - valuePairAssign(pList[0], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); -// taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); - } - } -} - -static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SSubsidiaryResInfo *pTagInfo, char *pTags, int16_t stage) { - SVariant val = {0}; - taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); - - tValuePair **pList = pInfo->res; - assert(pList != NULL); - - if (pInfo->num < maxLen) { - valuePairAssign(pList[pInfo->num], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - -// taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); - - pInfo->num++; - } else { - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i < pList[0]->v.i) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u < pList[0]->v.u) || - (IS_FLOAT_TYPE(type) && val.d < pList[0]->v.d)) { - valuePairAssign(pList[0], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); -// taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); - } - } -} - -static int32_t resAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = *(tValuePair **)pLeft; - tValuePair *pRightElem = *(tValuePair **)pRight; - - if (pLeftElem->timestamp == pRightElem->timestamp) { - return 0; - } else { - return pLeftElem->timestamp > pRightElem->timestamp ? 1 : -1; - } -} - -static int32_t resDescComparFn(const void *pLeft, const void *pRight) { return -resAscComparFn(pLeft, pRight); } - -static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = *(tValuePair **)pLeft; - tValuePair *pRightElem = *(tValuePair **)pRight; - - if (IS_FLOAT_TYPE(pLeftElem->v.nType)) { - if (pLeftElem->v.d == pRightElem->v.d) { - return 0; - } else { - return pLeftElem->v.d > pRightElem->v.d ? 1 : -1; - } - } else if (IS_SIGNED_NUMERIC_TYPE(pLeftElem->v.nType)){ - if (pLeftElem->v.i == pRightElem->v.i) { - return 0; - } else { - return pLeftElem->v.i > pRightElem->v.i ? 1 : -1; - } - } else { - if (pLeftElem->v.u == pRightElem->v.u) { - return 0; - } else { - return pLeftElem->v.u > pRightElem->v.u ? 1 : -1; - } - } -} - -static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } - -static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); - - tValuePair **tvp = pRes->res; - - int32_t step = QUERY_ASC_FORWARD_STEP; - int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes); - - switch (type) { - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - int32_t *output = (int32_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int32_t)tvp[i]->v.i; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - int64_t *output = (int64_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i]->v.i; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *output = (double *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - SET_DOUBLE_VAL(output, tvp[i]->v.d); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *output = (float *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (float)tvp[i]->v.d; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *output = (int16_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int16_t)tvp[i]->v.i; - } - break; - } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - int8_t *output = (int8_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int8_t)tvp[i]->v.i; - } - break; - } - default: { -// qError("top/bottom function not support data type:%d", pCtx->inputType); - return; - } - } - - // set the output timestamp of each record. -// TSKEY *output = pCtx->pTsOutput; -// for (int32_t i = 0; i < len; ++i, output += step) { -// *output = tvp[i]->timestamp; -// } - - // set the corresponding tag data for each record - // todo check malloc failure -// char **pData = taosMemoryCalloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); -// for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { -// pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; -// } - -// for (int32_t i = 0; i < len; ++i, output += step) { -// int16_t offset = 0; -// for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { -// memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes); -// offset += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; -// pData[j] += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; -// } -// } - -// taosMemoryFreeClear(pData); -} - -/* - * Parameters values: - * 1. param[0]: maximum allowable results - * 2. param[1]: order by type (time or value) - * 3. param[2]: asc/desc order - * - * top/bottom use the intermediate result buffer to keep the intermediate result - */ -static STopBotInfo *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - // only the first_stage_merge is directly written data into final output buffer - if (pCtx->stableQuery && pCtx->scanFlag != MERGE_STAGE) { - return (STopBotInfo*) pCtx->pOutput; - } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer - return GET_ROWCELL_INTERBUF(pResInfo); - } -} - - -/* - * keep the intermediate results during scan data blocks in the format of: - * +-----------------------------------+-------------one value pair-----------+------------next value pair-----------+ - * |-------------pointer area----------|----ts---+-----+-----n tags-----------|----ts---+-----+-----n tags-----------| - * +..[Value Pointer1][Value Pointer2].|timestamp|value|tags1|tags2|....|tagsn|timestamp|value|tags1|tags2|....|tagsn+ - */ -static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SqlFunctionCtx *pCtx) { - char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); - pTopBotInfo->res = (tValuePair**) tmp; -// tmp += POINTER_BYTES * pCtx->param[0].param.i; - -// size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; - -// for (int32_t i = 0; i < pCtx->param[0].param.i; ++i) { -// pTopBotInfo->res[i] = (tValuePair*) tmp; -// pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); -// tmp += size; -// } -} - -bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo == NULL) { - return true; - } - - STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx); - - // required number of results are not reached, continue load data block -// if (pTopBotInfo->num < pCtx->param[0].param.i) { -// return true; -// } - -// if ((void *)pTopBotInfo->res[0] != (void *)((char *)pTopBotInfo + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { -// buildTopBotStruct(pTopBotInfo, pCtx); -// } - - tValuePair **pRes = (tValuePair**) pTopBotInfo->res; - - if (pCtx->functionId == FUNCTION_TOP) { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(maxval) > pRes[0]->v.d; - case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(maxval) > pRes[0]->v.d; - default: - return true; - } - } else { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(minval) < pRes[0]->v.d; - case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(minval) < pRes[0]->v.d; - default: - return true; - } - } -} - -static bool top_bottom_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - STopBotInfo *pInfo = getTopBotOutputInfo(pCtx); - buildTopBotStruct(pInfo, pCtx); - return true; -} - -static void top_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - assert(pRes->num >= 0); - -// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { -// buildTopBotStruct(pRes, pCtx); -// } - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems++; - - // NOTE: Set the default timestamp if it is missing [todo refactor] - TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; -// do_top_function_add(pRes, (int32_t)pCtx->param[0].param.i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - // treat the result as only one result - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void top_func_merge(SqlFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); - - // construct the input data struct from binary data - buildTopBotStruct(pInput, pCtx); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - // the intermediate result is binary, we only use the output data type - for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->resDataInfo.type; -// do_top_function_add(pOutput, (int32_t)pCtx->param[0].param.i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, -// type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->scanFlag); - } - - SET_VAL(pCtx, pInput->num, pOutput->num); - - if (pOutput->num > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void bottom_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - -// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { -// buildTopBotStruct(pRes, pCtx); -// } - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems++; - // NOTE: Set the default timestamp if it is missing [todo refactor] - TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; -// do_bottom_function_add(pRes, (int32_t)pCtx->param[0].param.i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - // treat the result as only one result - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void bottom_func_merge(SqlFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); - - // construct the input data struct from binary data - buildTopBotStruct(pInput, pCtx); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - // the intermediate result is binary, we only use the output data type - for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->resDataInfo.type; -// do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].param.i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, -// &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->scanFlag); - } - - SET_VAL(pCtx, pInput->num, pOutput->num); - - if (pOutput->num > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void top_bottom_func_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - // data in temporary list is less than the required number of results, not enough qualified number of results - STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); - if (pRes->num == 0) { // no result -// assert(pResInfo->hasResult != DATA_SET_FLAG); - // TODO: - } - - GET_RES_INFO(pCtx)->numOfRes = pRes->num; - tValuePair **tvp = pRes->res; - - // user specify the order of output by sort the result according to timestamp - if (pCtx->param[1].param.i == PRIMARYKEY_TIMESTAMP_COL_ID) { - __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; - taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } else /*if (pCtx->param[1].param.i > PRIMARYKEY_TIMESTAMP_COL_ID)*/ { - __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; - taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } - - GET_TRUE_DATA_TYPE(); - copyTopBotRes(pCtx, type); - - doFinalizer(pCtx); -} - -/////////////////////////////////////////////////////////////////////////////////////////////// -static bool percentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - // in the first round, get the min-max value of all involved data - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); - SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX); - SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); - pInfo->numOfElems = 0; - - return true; -} - -static void percentile_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->scanFlag == REPEAT_SCAN && pInfo->stage == 0) { - pInfo->stage += 1; - - // all data are null, set it completed - if (pInfo->numOfElems == 0) { - pResInfo->complete = true; - - return; - } else { - pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); - } - } - - // the first stage, only acquire the min/max value - if (pInfo->stage == 0) { - if (pCtx->isAggSet) { - double tmin = 0.0, tmax = 0.0; - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_INT64_VAL(&pCtx->agg.min); - tmax = (double)GET_INT64_VAL(&pCtx->agg.max); - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - tmin = GET_DOUBLE_VAL(&pCtx->agg.min); - tmax = GET_DOUBLE_VAL(&pCtx->agg.max); - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_UINT64_VAL(&pCtx->agg.min); - tmax = (double)GET_UINT64_VAL(&pCtx->agg.max); - } else { - assert(true); - } - - if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) { - SET_DOUBLE_VAL(&pInfo->minval, tmin); - } - - if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) { - SET_DOUBLE_VAL(&pInfo->maxval, tmax); - } - - pInfo->numOfElems += (pCtx->size - pCtx->agg.numOfNull); - } else { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, data); - - if (v < GET_DOUBLE_VAL(&pInfo->minval)) { - SET_DOUBLE_VAL(&pInfo->minval, v); - } - - if (v > GET_DOUBLE_VAL(&pInfo->maxval)) { - SET_DOUBLE_VAL(&pInfo->maxval, v); - } - - pInfo->numOfElems += 1; - } - } - - return; - } - - // the second stage, calculate the true percentile value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - tMemBucketPut(pInfo->pMemBucket, data, 1); - } - - SET_VAL(pCtx, notNullElems, 1); - //pResInfo->hasResult = DATA_SET_FLAG; -} - -static void percentile_finalizer(SqlFunctionCtx *pCtx) { -// double v = pCtx->param[0].param.nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].param.i : pCtx->param[0].param.d; - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); - - tMemBucket * pMemBucket = ppInfo->pMemBucket; - if (pMemBucket == NULL || pMemBucket->total == 0) { // check for null - assert(ppInfo->numOfElems == 0); - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else { -// SET_DOUBLE_VAL((double *)pCtx->pOutput, getPercentile(pMemBucket, v)); - } - - tMemBucketDestroy(pMemBucket); - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////// -static void buildHistogramInfo(SAPercentileInfo* pInfo) { - pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo)); - pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); -} - -static SAPercentileInfo *getAPerctInfo(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo* pInfo = NULL; - - if (pCtx->stableQuery && pCtx->scanFlag != MERGE_STAGE) { - pInfo = (SAPercentileInfo*) pCtx->pOutput; - } else { - pInfo = GET_ROWCELL_INTERBUF(pResInfo); - } - - buildHistogramInfo(pInfo); - return pInfo; -} - -static bool apercentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - - char *tmp = (char *)pInfo + sizeof(SAPercentileInfo); - pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN); - return true; -} - -static void apercentile_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - - assert(pInfo->pHisto->elems != NULL); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, data); - tHistogramAdd(&pInfo->pHisto, v); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void apercentile_func_merge(SqlFunctionCtx *pCtx) { - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); - - pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); - pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - - if (pInput->pHisto->numOfElems <= 0) { - return; - } - - SAPercentileInfo *pOutput = getAPerctInfo(pCtx); - SHistogramInfo *pHisto = pOutput->pHisto; - - if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - } else { - //TODO(dengyihao): avoid memcpy - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - tHistogramDestroy(&pRes); - } - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - SET_VAL(pCtx, 1, 1); -} - -static void apercentile_finalizer(SqlFunctionCtx *pCtx) { - double v = (pCtx->param[0].param.nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].param.i : pCtx->param[0].param.d; - - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->scanFlag == MERGE_STAGE) { -// if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null -// assert(pOutput->pHisto->numOfElems > 0); -// -// double ratio[] = {v}; -// double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); -// -// memcpy(pCtx->pOutput, res, sizeof(double)); -// taosMemoryFree(res); -// } else { -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// return; -// } - } else { - if (pOutput->pHisto->numOfElems > 0) { - double ratio[] = {v}; - - double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - memcpy(pCtx->pOutput, res, sizeof(double)); - taosMemoryFree(res); - } else { // no need to free - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - } - - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////// -static bool leastsquares_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // 2*3 matrix -// pInfo->startVal = pCtx->param[0].param.d; - return true; -} - -#define LEASTSQR_CAL(p, x, y, index, step) \ - do { \ - (p)[0][0] += (double)(x) * (x); \ - (p)[0][1] += (double)(x); \ - (p)[0][2] += (double)(x) * (y)[index]; \ - (p)[1][2] += (y)[index]; \ - (x) += step; \ - } while (0) - -#define LEASTSQR_CAL_LOOP(ctx, param, x, y, tsdbType, n, step) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&(y)[i], tsdbType)) { \ - continue; \ - } \ - (n)++; \ - LEASTSQR_CAL(param, x, y, i, step); \ - } - -static void leastsquares_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - double(*param)[3] = pInfo->mat; - double x = pInfo->startVal; - - void *pData = GET_INPUT_DATA_LIST(pCtx); - - int32_t numOfElem = 0; - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *p = pData; - // LEASTSQR_CAL_LOOP(pCtx, param, pParamData, p); - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) p, pCtx->inputType)) { - continue; - } - - param[0][0] += x * x; - param[0][1] += x; - param[0][2] += x * p[i]; - param[1][2] += p[i]; - - x += pCtx->param[1].param.d; - numOfElem++; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - }; - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - int8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - } - - pInfo->startVal = x; - pInfo->num += numOfElem; - - if (pInfo->num > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } - - SET_VAL(pCtx, numOfElem, 1); -} - -static void leastsquares_finalizer(SqlFunctionCtx *pCtx) { - // no data in query - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pInfo->num == 0) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - double(*param)[3] = pInfo->mat; - - param[1][1] = (double)pInfo->num; - param[1][0] = param[0][1]; - - param[0][0] -= param[1][0] * (param[0][1] / param[1][1]); - param[0][2] -= param[1][2] * (param[0][1] / param[1][1]); - param[0][1] = 0; - param[1][2] -= param[0][2] * (param[1][0] / param[0][0]); - param[1][0] = 0; - param[0][2] /= param[0][0]; - - param[1][2] /= param[1][1]; - - int32_t maxOutputSize = AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE; - size_t n = snprintf(varDataVal(pCtx->pOutput), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}", - param[0][2], param[1][2]); - - varDataSetLen(pCtx->pOutput, n); - doFinalizer(pCtx); -} - -static void date_col_output_function(SqlFunctionCtx *pCtx) { - SET_VAL(pCtx, pCtx->size, 1); - *(int64_t *)(pCtx->pOutput) = pCtx->startTs; -} - -static void col_project_function(SqlFunctionCtx *pCtx) { - // the number of output rows should not affect the final number of rows, so set it to be 0 - if (pCtx->numOfParams == 2) { - return; - } - - // only one row is required. -// if (pCtx->param[0].param.i == 1) { -// SET_VAL(pCtx, pCtx->size, 1); -// } else { -// INC_INIT_VAL(pCtx, pCtx->size); -// } - - char *pData = GET_INPUT_DATA_LIST(pCtx); - if (pCtx->order == TSDB_ORDER_ASC) { -// int32_t numOfRows = (pCtx->param[0].param.i == 1)? 1:pCtx->size; -// memcpy(pCtx->pOutput, pData, (size_t) numOfRows * pCtx->inputBytes); - } else { - for(int32_t i = 0; i < pCtx->size; ++i) { - memcpy(pCtx->pOutput + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, - pCtx->inputBytes); - } - } -} - -/** - * only used for tag projection query in select clause - * @param pCtx - * @return - */ -static void tag_project_function(SqlFunctionCtx *pCtx) { - INC_INIT_VAL(pCtx, pCtx->size); - - assert(pCtx->inputBytes == pCtx->resDataInfo.bytes); - - taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->resDataInfo.type, true); - char* data = pCtx->pOutput; - pCtx->pOutput += pCtx->resDataInfo.bytes; - - // directly copy from the first one - for (int32_t i = 1; i < pCtx->size; ++i) { - memmove(pCtx->pOutput, data, pCtx->resDataInfo.bytes); - pCtx->pOutput += pCtx->resDataInfo.bytes; - } -} - -/** - * used in group by clause. when applying group by tags, the tags value is - * assign by using tag function. - * NOTE: there is only ONE output for ONE query range - * @param pCtx - * @return - */ -static void copy_function(SqlFunctionCtx *pCtx); - -static void tag_function(SqlFunctionCtx *pCtx) { - SET_VAL(pCtx, 1, 1); - if (pCtx->scanFlag == MERGE_STAGE) { - copy_function(pCtx); - } else { - taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->resDataInfo.type, true); - } -} - -static void copy_function(SqlFunctionCtx *pCtx) { - SET_VAL(pCtx, pCtx->size, 1); - - char *pData = GET_INPUT_DATA_LIST(pCtx); - assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); -} - -enum { - INITIAL_VALUE_NOT_ASSIGNED = 0, -}; - -static bool diff_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - // diff function require the value is set to -1 - pCtx->param[1].param.nType = INITIAL_VALUE_NOT_ASSIGNED; - return false; -} - -static bool deriv_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - // diff function require the value is set to -1 - SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResultInfo); - -// pDerivInfo->ignoreNegative = pCtx->param[1].param.i; - pDerivInfo->prevTs = -1; -// pDerivInfo->tsWindow = pCtx->param[0].param.i; - pDerivInfo->valueSet = false; - return false; -} - -static void deriv_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo); - - void *data = GET_INPUT_DATA_LIST(pCtx); - - int32_t notNullElems = 0; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - - TSKEY *pTimestamp = NULL;//pCtx->pTsOutput; - TSKEY *tsList = GET_TS_LIST(pCtx); - - double *pOutput = (double *)pCtx->pOutput; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *pData = (int32_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - SET_DOUBLE_VAL(pOutput, ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs)); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - - break; - }; - - case TSDB_DATA_TYPE_BIGINT: { - int64_t *pData = (int64_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = (double) pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *pData = (double *)data; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_FLOAT: { - float *pData = (float *)data; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *pData = (int16_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t *pData = (int8_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - default: - assert(0); -// qError("error input type"); - } - - GET_RES_INFO(pCtx)->numOfRes += notNullElems; -} - -// TODO difference in date column -static void diff_function(SqlFunctionCtx *pCtx) { - void *data = GET_INPUT_DATA_LIST(pCtx); - bool isFirstBlock = (pCtx->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED); - - int32_t notNullElems = 0; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - - TSKEY* pTimestamp = NULL;//pCtx->pTsOutput; - TSKEY* tsList = GET_TS_LIST(pCtx); - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *pData = (int32_t *)data; - int32_t *pOutput = (int32_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int32_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - }; - case TSDB_DATA_TYPE_BIGINT: { - int64_t *pData = (int64_t *)data; - int64_t *pOutput = (int64_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = pData[i] - pCtx->param[1].param.i; // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *pData = (double *)data; - double *pOutput = (double *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].param.d); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.d = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *pData = (float *)data; - float *pOutput = (float *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (float)(pData[i] - pCtx->param[1].param.d); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.d = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *pData = (int16_t *)data; - int16_t *pOutput = (int16_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int16_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t *pData = (int8_t *)data; - int8_t *pOutput = (int8_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int8_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - default: - assert(0); -// qError("error input type"); - } - - // initial value is not set yet - if (pCtx->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { - /* - * 1. current block and blocks before are full of null - * 2. current block may be null value - */ - assert(pCtx->hasNull); - } else { - int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems; - - GET_RES_INFO(pCtx)->numOfRes += forwardStep; - } -} - -#if 0 -char *getArithColumnData(void *param, const char* name, int32_t colId) { - SScalarFunctionSupport *pSupport = (SScalarFunctionSupport *)param; - - int32_t index = -1; - for (int32_t i = 0; i < pSupport->numOfCols; ++i) { - if (colId == pSupport->colList[i].colId) { - index = i; - break; - } - } - - assert(index >= 0); - return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; -} -#endif - -static void arithmetic_function(SqlFunctionCtx *pCtx) { - GET_RES_INFO(pCtx)->numOfRes += pCtx->size; - //SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; - -// SScalarParam output = {0}; -// output.data = pCtx->pOutput; - - //evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); -} - -#define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ - { \ - type *inputData = (type *)data; \ - for (int32_t i = 0; i < elemCnt; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&inputData[i], tsdbType)) { \ - continue; \ - } \ - if (inputData[i] < minOutput) { \ - minOutput = (double)inputData[i]; \ - } \ - if (inputData[i] > maxOutput) { \ - maxOutput = (double)inputData[i]; \ - } \ - numOfNotNullElem++; \ - } \ - } - -///////////////////////////////////////////////////////////////////////////////// -static bool spread_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // this is the server-side setup function in client-side, the secondary merge do not need this procedure - if (pCtx->scanFlag == MERGE_STAGE) { -// pCtx->param[0].param.d = DBL_MAX; -// pCtx->param[3].param.d = -DBL_MAX; - } else { - pInfo->min = DBL_MAX; - pInfo->max = -DBL_MAX; - } - - return true; -} - -static void spread_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - int32_t numOfElems = 0; - - // todo : opt with pre-calculated result - // column missing cause the hasNull to be true - if (pCtx->isAggSet) { - numOfElems = pCtx->size - pCtx->agg.numOfNull; - - // all data are null in current data block, ignore current data block - if (numOfElems == 0) { - goto _spread_over; - } - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType) || IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType) || - (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { - if (pInfo->min > pCtx->agg.min) { - pInfo->min = (double)pCtx->agg.min; - } - - if (pInfo->max < pCtx->agg.max) { - pInfo->max = (double)pCtx->agg.max; - } - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - if (pInfo->min > GET_DOUBLE_VAL((const char *)&(pCtx->agg.min))) { - pInfo->min = GET_DOUBLE_VAL((const char *)&(pCtx->agg.min)); - } - - if (pInfo->max < GET_DOUBLE_VAL((const char *)&(pCtx->agg.max))) { - pInfo->max = GET_DOUBLE_VAL((const char *)&(pCtx->agg.max)); - } - } - - goto _spread_over; - } - - void *pData = GET_INPUT_DATA_LIST(pCtx); - numOfElems = 0; - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int8_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int16_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int32_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT || pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int64_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, double, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, float, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint8_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint16_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint32_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint64_t, pCtx->inputType, numOfElems); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == numOfElems); - } - - _spread_over: - SET_VAL(pCtx, numOfElems, 1); - - if (numOfElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - pInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); - } -} - -/* - * here we set the result value back to the intermediate buffer, to apply the finalize the function - * the final result is generated in spread_function_finalizer - */ -void spread_func_merge(SqlFunctionCtx *pCtx) { - SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); - if (pData->hasResult != DATA_SET_FLAG) { - return; - } - -// if (pCtx->param[0].param.d > pData->min) { -// pCtx->param[0].param.d = pData->min; -// } - -// if (pCtx->param[3].param.d < pData->max) { -// pCtx->param[3].param.d = pData->max; -// } - -// GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -void spread_function_finalizer(SqlFunctionCtx *pCtx) { - /* - * here we do not check the input data types, because in case of metric query, - * the type of intermediate data is binary - */ - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->scanFlag == MERGE_STAGE) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - -// if (pResInfo->hasResult != DATA_SET_FLAG) { -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// return; -// } - -// SET_DOUBLE_VAL((double *)pCtx->pOutput, pCtx->param[3].param.d - pCtx->param[0].param.d); - } else { - assert(IS_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); - - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->max - pInfo->min); - } - - GET_RES_INFO(pCtx)->numOfRes = 1; // todo add test case - doFinalizer(pCtx); -} - - -/** - * param[1]: start time - * param[2]: end time - * @param pCtx - */ -static bool twa_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->p.key = INT64_MIN; - pInfo->win = TSWINDOW_INITIALIZER; - return true; -} - -static double twa_get_area(SPoint1 s, SPoint1 e) { - if ((s.val >= 0 && e.val >= 0)|| (s.val <=0 && e.val <= 0)) { - return (s.val + e.val) * (e.key - s.key) / 2; - } - - double x = (s.key * e.val - e.key * s.val)/(e.val - s.val); - double val = (s.val * (x - s.key) + e.val * (e.key - x)) / 2; - return val; -} - -static int32_t twa_function_impl(SqlFunctionCtx* pCtx, int32_t index, int32_t size) { - int32_t notNullElems = 0; - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *tsList = GET_TS_LIST(pCtx); - - int32_t i = index; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - SPoint1* last = &pInfo->p; - - if (pCtx->start.key != INT64_MIN) { - assert((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || - (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); - - assert(last->key == INT64_MIN); - - last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); - - pInfo->dOutput += twa_get_area(pCtx->start, *last); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pCtx->start.key; - notNullElems++; - i += step; - } else if (pInfo->p.key == INT64_MIN) { - last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = last->key; - notNullElems++; - i += step; - } - - // calculate the value of - switch(pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t *val = (uint8_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t *val = (uint16_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t *val = (uint32_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t *val = (uint64_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double) val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - default: assert(0); - } - - // the last interpolated time window value - if (pCtx->end.key != INT64_MIN) { - pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end); - pInfo->p = pCtx->end; - } - - pInfo->win.ekey = pInfo->p.key; - return notNullElems; -} - -static void twa_function(SqlFunctionCtx *pCtx) { - void *data = GET_INPUT_DATA_LIST(pCtx); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // skip null value - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); - while (pCtx->hasNull && i < pCtx->size && i >= 0 && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { - i += step; - } - - int32_t notNullElems = 0; - if (i >= 0 && i < pCtx->size) { - notNullElems = twa_function_impl(pCtx, i, pCtx->size); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } - - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, pInfo, sizeof(STwaInfo)); - } -} - -/* - * To copy the input to interResBuf to avoid the input buffer space be over writen - * by next input data. The TWA function only applies to each table, so no merge procedure - * is required, we simply copy to the resut ot interResBuffer. - */ -void twa_function_copy(SqlFunctionCtx *pCtx) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); -// pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; -} - -void twa_function_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); - if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - return; - } - -// assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult); - if (pInfo->win.ekey == pInfo->win.skey) { - SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->p.val); - } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput , pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey)); - } - - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -/** - * - * @param pCtx - */ - -static void interp_function_impl(SqlFunctionCtx *pCtx) { - int32_t type = (int32_t) pCtx->param[2].param.i; - if (type == TSDB_FILL_NONE) { - return; - } - - bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); - - if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - *(TSKEY *)pCtx->pOutput = pCtx->startTs; - } else if (type == TSDB_FILL_NULL) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else if (type == TSDB_FILL_SET_VALUE) { -// taosVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); - } else { - if (pCtx->start.key != INT64_MIN && ((ascQuery && pCtx->start.key <= pCtx->startTs && pCtx->end.key >= pCtx->startTs) || ((!ascQuery) && pCtx->start.key >= pCtx->startTs && pCtx->end.key <= pCtx->startTs))) { - if (type == TSDB_FILL_PREV) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val); - } else { - assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->resDataInfo.bytes, pCtx->inputType); - } - } else if (type == TSDB_FILL_NEXT) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->end.val); - } else { - assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->resDataInfo.bytes, pCtx->inputType); - } - } else if (type == TSDB_FILL_LINEAR) { - SPoint point1 = {.key = pCtx->start.key, .val = &pCtx->start.val}; - SPoint point2 = {.key = pCtx->end.key, .val = &pCtx->end.val}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; - - int32_t srcType = pCtx->inputType; - if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? - if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } else { -// taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); - } - } else { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } - } - } else { - // no data generated yet - if (pCtx->size < 1) { - return; - } - - // check the timestamp in input buffer - TSKEY skey = GET_TS_DATA(pCtx, 0); - - if (type == TSDB_FILL_PREV) { - if ((ascQuery && skey > pCtx->startTs) || ((!ascQuery) && skey < pCtx->startTs)) { - return; - } - - if (pCtx->size > 1) { - TSKEY ekey = GET_TS_DATA(pCtx, 1); - if ((ascQuery && ekey > skey && ekey <= pCtx->startTs) || - ((!ascQuery) && ekey < skey && ekey >= pCtx->startTs)){ - skey = ekey; - } - } -// assignVal(pCtx->pOutput, pCtx->pInput, pCtx->resDataInfo.bytes, pCtx->inputType); - } else if (type == TSDB_FILL_NEXT) { - TSKEY ekey = skey; - char* val = NULL; - - if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) { - if (pCtx->size > 1) { - ekey = GET_TS_DATA(pCtx, 1); - if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) { - return; - } - - val = ((char*)pCtx->pInput) + pCtx->inputBytes; - } else { - return; - } - } else { - val = (char*)pCtx->pInput; - } - - assignVal(pCtx->pOutput, val, pCtx->resDataInfo.bytes, pCtx->inputType); - } else if (type == TSDB_FILL_LINEAR) { - if (pCtx->size <= 1) { - return; - } - - TSKEY ekey = GET_TS_DATA(pCtx, 1); - - // no data generated yet - if ((ascQuery && !(skey <= pCtx->startTs && ekey >= pCtx->startTs)) - || ((!ascQuery) && !(skey >= pCtx->startTs && ekey <= pCtx->startTs))) { - return; - } - - char *start = GET_INPUT_DATA(pCtx, 0); - char *end = GET_INPUT_DATA(pCtx, 1); - - SPoint point1 = {.key = skey, .val = start}; - SPoint point2 = {.key = ekey, .val = end}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; - - int32_t srcType = pCtx->inputType; - if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? - if (isNull(start, srcType) || isNull(end, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } else { -// taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, srcType); - } - } else { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } - } - } - } - - SET_VAL(pCtx, 1, 1); -} - -static void interp_function(SqlFunctionCtx *pCtx) { - // at this point, the value is existed, return directly - if (pCtx->size > 0) { - bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); - TSKEY key; - char *pData; - int32_t typedData = 0; - - if (ascQuery) { - key = GET_TS_DATA(pCtx, 0); - pData = GET_INPUT_DATA(pCtx, 0); - } else { - key = pCtx->start.key; - if (key == INT64_MIN) { - key = GET_TS_DATA(pCtx, 0); - pData = GET_INPUT_DATA(pCtx, 0); - } else { - if (!(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL)) { - pData = pCtx->start.ptr; - } else { - typedData = 1; - pData = (char *)&pCtx->start.val; - } - } - } - - //if (key == pCtx->startTs && (ascQuery || !(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL))) { - if (key == pCtx->startTs) { - if (typedData) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, *(double *)pData); - } else { - assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); - } - - SET_VAL(pCtx, 1, 1); - } else { - interp_function_impl(pCtx); - } - } else { //no qualified data rows and interpolation is required - interp_function_impl(pCtx); - } -} - -static bool ts_comp_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; // not initialized since it has been initialized - } - - STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->pTSBuf = tsBufCreate(false, pCtx->order); - pInfo->pTSBuf->tsOrder = pCtx->order; - return true; -} - -static void ts_comp_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; - - const char *input = GET_INPUT_DATA_LIST(pCtx); - - // primary ts must be existed, so no need to check its existance - if (pCtx->order == TSDB_ORDER_ASC) { -// tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].param.i, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); - } else { - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *d = GET_INPUT_DATA(pCtx, i); -// tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].param.i, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); - } - } - - SET_VAL(pCtx, pCtx->size, 1); - //pResInfo->hasResult = DATA_SET_FLAG; -} - -static void ts_comp_finalize(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - STSBuf * pTSbuf = pInfo->pTSBuf; - - tsBufFlush(pTSbuf); -// qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); - - // TODO refactor transfer ownership of current file - *(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile; - - pResInfo->complete = true; - - // get the file size - int64_t file_size; - if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) { - pResInfo->numOfRes = (uint32_t )file_size; - } - - pTSbuf->remainOpen = true; - tsBufDestroy(pTSbuf); - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////////////// -// rate functions -static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) { - if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey) || - (pRateInfo->firstKey >= pRateInfo->lastKey)) { - return 0.0; - } - - double diff = 0; - if (pRateInfo->isIRate) { - // If the previous value of the last is greater than the last value, only keep the last point instead of the delta - // value between two values. - diff = pRateInfo->lastValue; - if (diff >= pRateInfo->firstValue) { - diff -= pRateInfo->firstValue; - } - } else { - diff = pRateInfo->correctionValue + pRateInfo->lastValue - pRateInfo->firstValue; - if (diff <= 0) { - return 0; - } - } - - int64_t duration = pRateInfo->lastKey - pRateInfo->firstKey; - if (duration == 0) { - return 0; - } - - return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; -} - -static bool rate_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SRateInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->correctionValue = 0; - pInfo->firstKey = INT64_MIN; - pInfo->lastKey = INT64_MIN; - pInfo->firstValue = (double) INT64_MIN; - pInfo->lastValue = (double) INT64_MIN; - - pInfo->hasResult = 0; - pInfo->isIRate = (pCtx->functionId == FUNCTION_IRATE); - return true; -} - -static void rate_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - -// qDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *pData = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { -// qDebug("%p rate_function() index of null data:%d", pCtx, i); - continue; - } - - notNullElems++; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; - } - - if (INT64_MIN == pRateInfo->lastValue) { - pRateInfo->lastValue = v; - } else if (v < pRateInfo->lastValue) { - pRateInfo->correctionValue += pRateInfo->lastValue; - } - - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[i]; - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pRateInfo->hasResult = DATA_SET_FLAG; -// pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - -static void rate_func_copy(SqlFunctionCtx *pCtx) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); -// pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; -} - -static void rate_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pRateInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - return; - } - -// SET_DOUBLE_VAL((double*) pCtx->pOutput, do_calc_rate(pRateInfo, (double) TSDB_TICK_PER_SECOND(pCtx->param[0].param.i))); - - // cannot set the numOfIteratedElems again since it is set during previous iteration - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; - - doFinalizer(pCtx); -} - -static void irate_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *pData = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - continue; - } - - notNullElems++; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if ((INT64_MIN == pRateInfo->lastKey) || primaryKey[i] > pRateInfo->lastKey) { - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[i]; - continue; - } - - if ((INT64_MIN == pRateInfo->firstKey) || primaryKey[i] > pRateInfo->firstKey) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; - break; - } - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pRateInfo->hasResult = DATA_SET_FLAG; -// pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - -static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDistInfo* pDist) { - SBufferReader br = tbufInitReader(data, len, false); - - pDist->numOfTables = tbufReadUint32(&br); - pDist->numOfFiles = tbufReadUint16(&br); - pDist->totalSize = tbufReadUint64(&br); - pDist->totalRows = tbufReadUint64(&br); - pDist->maxRows = tbufReadInt32(&br); - pDist->minRows = tbufReadInt32(&br); - pDist->numOfInmemRows = tbufReadUint32(&br); - pDist->numOfSmallBlocks = tbufReadUint32(&br); - int64_t numSteps = tbufReadUint64(&br); - - bool comp = tbufReadUint8(&br); - uint32_t compLen = tbufReadUint32(&br); - - size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo)); - - char* outputBuf = NULL; - if (comp) { - outputBuf = taosMemoryMalloc(originalLen); - - size_t actualLen = compLen; - const char* compStr = tbufReadBinary(&br, &actualLen); - - int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, - (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); - assert(orignalLen == numSteps *sizeof(SFileBlockInfo)); - } else { - outputBuf = (char*) tbufReadBinary(&br, &originalLen); - } - - pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); - if (comp) { - taosMemoryFreeClear(outputBuf); - } -} - -static void blockInfo_func(SqlFunctionCtx* pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); - - int32_t len = *(int32_t*) pCtx->pInput; - blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); -// pDist->rowSize = (uint16_t)pCtx->param[0].param.i; - - memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); - - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; -} - -static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDistInfo* pSrc) { - STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); - assert(pDist != NULL && pSrc != NULL); - - pDist->numOfTables += pSrc->numOfTables; - pDist->numOfInmemRows += pSrc->numOfInmemRows; - pDist->numOfSmallBlocks += pSrc->numOfSmallBlocks; - pDist->numOfFiles += pSrc->numOfFiles; - pDist->totalSize += pSrc->totalSize; - pDist->totalRows += pSrc->totalRows; - -// if (pResInfo->hasResult == DATA_SET_FLAG) { -// pDist->maxRows = TMAX(pDist->maxRows, pSrc->maxRows); -// pDist->minRows = TMIN(pDist->minRows, pSrc->minRows); -// } else { - pDist->maxRows = pSrc->maxRows; - pDist->minRows = pSrc->minRows; - - int32_t maxSteps = TSDB_MAX_MAXROWS_FBLOCK/TSDB_BLOCK_DIST_STEP_ROWS; - if (TSDB_MAX_MAXROWS_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { - ++maxSteps; - } - pDist->dataBlockInfos = taosArrayInit(maxSteps, sizeof(SFileBlockInfo)); - taosArraySetSize(pDist->dataBlockInfos, maxSteps); -// } - - size_t steps = taosArrayGetSize(pSrc->dataBlockInfos); - for (int32_t i = 0; i < steps; ++i) { - int32_t srcNumBlocks = ((SFileBlockInfo*)taosArrayGet(pSrc->dataBlockInfos, i))->numBlocksOfStep; - SFileBlockInfo* blockInfo = (SFileBlockInfo*)taosArrayGet(pDist->dataBlockInfos, i); - blockInfo->numBlocksOfStep += srcNumBlocks; - } -} - -void block_func_merge(SqlFunctionCtx* pCtx) { - STableBlockDistInfo info = {0}; - int32_t len = *(int32_t*) pCtx->pInput; - blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - mergeTableBlockDist(pResInfo, &info); - taosArrayDestroy(info.dataBlockInfos); - - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; -} - -void getPercentiles(STableBlockDistInfo *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, - double* percents, int32_t* percentiles) { - if (totalBlocks == 0) { - for (int32_t i = 0; i < numOfPercents; ++i) { - percentiles[i] = 0; - } - return; - } - - SArray *blocksInfos = pTableBlockDist->dataBlockInfos; - size_t numSteps = taosArrayGetSize(blocksInfos); - size_t cumulativeBlocks = 0; - - int percentIndex = 0; - for (int32_t indexStep = 0; indexStep < numSteps; ++indexStep) { - int32_t numStepBlocks = ((SFileBlockInfo *)taosArrayGet(blocksInfos, indexStep))->numBlocksOfStep; - if (numStepBlocks == 0) continue; - cumulativeBlocks += numStepBlocks; - - while (percentIndex < numOfPercents) { - double blockRank = totalBlocks * percents[percentIndex]; - if (blockRank <= cumulativeBlocks) { - percentiles[percentIndex] = indexStep; - ++percentIndex; - } else { - break; - } - } - } - - for (int32_t i = 0; i < numOfPercents; ++i) { - percentiles[i] = (percentiles[i]+1) * TSDB_BLOCK_DIST_STEP_ROWS - TSDB_BLOCK_DIST_STEP_ROWS/2; - } -} - -void generateBlockDistResult(STableBlockDistInfo *pTableBlockDist, char* result) { - if (pTableBlockDist == NULL) { - return; - } - - SArray* blockInfos = pTableBlockDist->dataBlockInfos; - uint64_t totalRows = pTableBlockDist->totalRows; - size_t numSteps = taosArrayGetSize(blockInfos); - int64_t totalBlocks = 0; - int64_t min = -1, max = -1, avg = 0; - - for (int32_t i = 0; i < numSteps; i++) { - SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t blocks = blockInfo->numBlocksOfStep; - totalBlocks += blocks; - } - - avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; - min = totalBlocks > 0 ? pTableBlockDist->minRows : 0; - max = totalBlocks > 0 ? pTableBlockDist->maxRows : 0; - - double stdDev = 0; - if (totalBlocks > 0) { - double variance = 0; - for (int32_t i = 0; i < numSteps; i++) { - SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t blocks = blockInfo->numBlocksOfStep; - int32_t rows = (i * TSDB_BLOCK_DIST_STEP_ROWS + TSDB_BLOCK_DIST_STEP_ROWS / 2); - variance += blocks * (rows - avg) * (rows - avg); - } - variance = variance / totalBlocks; - stdDev = sqrt(variance); - } - - double percents[] = {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99}; - int32_t percentiles[] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; - assert(sizeof(percents)/sizeof(double) == sizeof(percentiles)/sizeof(int32_t)); - getPercentiles(pTableBlockDist, totalBlocks, sizeof(percents)/sizeof(double), percents, percentiles); - - uint64_t totalLen = pTableBlockDist->totalSize; - int32_t rowSize = pTableBlockDist->rowSize; - int32_t smallBlocks = pTableBlockDist->numOfSmallBlocks; - double compRatio = (totalRows>0) ? ((double)(totalLen)/(rowSize*totalRows)) : 1; - int sz = sprintf(result + VARSTR_HEADER_SIZE, - "summary: \n\t " - "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t " - "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t " - "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t " - "Rows=[%"PRIu64"], Blocks=[%"PRId64"], SmallBlocks=[%d], Size=[%.3f(Kb)] Comp=[%.2f]\n\t " - "RowsInMem=[%d] \n\t", - percentiles[0], percentiles[1], percentiles[2], percentiles[3], percentiles[4], percentiles[5], - percentiles[6], percentiles[7], percentiles[8], percentiles[9], percentiles[10], percentiles[11], - min, max, avg, stdDev, - totalRows, totalBlocks, smallBlocks, totalLen/1024.0, compRatio, - pTableBlockDist->numOfInmemRows); - varDataSetLen(result, sz); - UNUSED(sz); -} - -void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); - -// pDist->rowSize = (uint16_t)pCtx->param[0].param.i; - generateBlockDistResult(pDist, pCtx->pOutput); - - if (pDist->dataBlockInfos != NULL) { - taosArrayDestroy(pDist->dataBlockInfos); - pDist->dataBlockInfos = NULL; - } - - // cannot set the numOfIteratedElems again since it is set during previous iteration - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; - - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////////////////// -/* - * function compatible list. - * tag and ts are not involved in the compatibility check - * - * 1. functions that are not simultaneously present with any other functions. e.g., diff/ts_z/top/bottom - * 2. functions that are only allowed to be present only with same functions. e.g., last_row, interp - * 3. functions that are allowed to be present with other functions. - * e.g., count/sum/avg/min/max/stddev/percentile/apercentile/first/last... - * - */ -int32_t functionCompatList[] = { - // count, sum, avg, min, max, stddev, percentile, apercentile, first, last - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // last_row,top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_comp - 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, - // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, stddev_dst, interp rate irate - 1, 1, 1, 1, -1, 1, 1, 1, 5, 1, 1, - // tid_tag, derivative, blk_info - 6, 8, 7, -}; -#endif diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c deleted file mode 100644 index 039e0a9dfc..0000000000 --- a/source/libs/function/src/texpr.c +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "function.h" -#include "os.h" - -#include "texception.h" -#include "tmsg.h" - -static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)); - -void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { - if (pNode == NULL) { - return; - } - - if (pNode->nodeType == TEXPR_BINARYEXPR_NODE || pNode->nodeType == TEXPR_UNARYEXPR_NODE) { - doExprTreeDestroy(&pNode, fp); - } - taosMemoryFree(pNode); -} - -static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { - if (*pExpr == NULL) { - return; - } - taosMemoryFree(*pExpr); - *pExpr = NULL; -} - -// TODO: these three functions should be made global -static void* exception_calloc(size_t nmemb, size_t size) { - void* p = taosMemoryCalloc(nmemb, size); - if (p == NULL) { - THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - return p; -} - -static void* exception_malloc(size_t size) { - void* p = taosMemoryMalloc(size); - if (p == NULL) { - THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - return p; -} - -static UNUSED_FUNC char* exception_strdup(const char* str) { - char* p = strdup(str); - if (p == NULL) { - THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - return p; -} - diff --git a/source/libs/function/src/tfunctionInt.c b/source/libs/function/src/tfunctionInt.c new file mode 100644 index 0000000000..85719e428b --- /dev/null +++ b/source/libs/function/src/tfunctionInt.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taosdef.h" +#include "tmsg.h" +#include "thash.h" +#include "ttypes.h" + +#include "function.h" +#include "tbuffer.h" +#include "tcompression.h" +#include "tdatablock.h" +#include "tfunctionInt.h" +#include "thistogram.h" +#include "tpercentile.h" +#include "ttszip.h" +#include "tudf.h" + +void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { + pCell->initialized = false; +} + +int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock) { + int32_t maxRows = 0; + + for (int32_t j = 0; j < num; ++j) { +#if 0 + int32_t id = pCtx[j].functionId; + + /* + * ts, tag, tagprj function can not decide the output number of current query + * the number of output result is decided by main output + */ + if (id == FUNCTION_TS || id == FUNCTION_TAG || id == FUNCTION_TAGPRJ) { + continue; + } +#endif + SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); + if (pResInfo != NULL && maxRows < pResInfo->numOfRes) { + maxRows = pResInfo->numOfRes; + } + } + + assert(maxRows >= 0); + + blockDataEnsureCapacity(pResBlock, maxRows); + for(int32_t i = 0; i < num; ++i) { + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); + + SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[i]); + if (pResInfo->numOfRes == 0) { + for(int32_t j = 0; j < pResInfo->numOfRes; ++j) { + colDataAppend(pCol, j, NULL, true); // TODO add set null data api + } + } else { + for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { + colDataAppend(pCol, j, GET_ROWCELL_INTERBUF(pResInfo), false); + } + } + } + + pResBlock->info.rows = maxRows; + return maxRows; +} + +bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { + assert(pEntry != NULL); + return pEntry->complete; +} + +bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry) { + return pEntry->initialized; +} From cfb61d665d9b743fd9dcf78ef2d9b17d0002b2f0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 10 Aug 2022 20:44:48 +0800 Subject: [PATCH 88/96] fix: row iter --- include/common/trow.h | 2 +- source/common/src/trow.c | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index a1db2ef708..ad9f52390e 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -256,7 +256,7 @@ static FORCE_INLINE void *tdGetBitmapAddrTp(STSRow *pRow, uint32_t flen) { static FORCE_INLINE void *tdGetBitmapAddrKv(STSRow *pRow, col_id_t nKvCols) { // The primary TS key is stored separatedly and is Norm value, thus should minus 1 firstly - return POINTER_SHIFT(TD_ROW_COL_IDX(pRow), nKvCols * sizeof(SKvRowIdx)); + return POINTER_SHIFT(TD_ROW_COL_IDX(pRow), (--nKvCols) * sizeof(SKvRowIdx)); } void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols); int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index b13e5f1ca8..d81fbf4629 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -427,7 +427,7 @@ bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVa tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); return true; } - int16_t nCols = tdRowGetNCols(pRow); + int16_t nCols = tdRowGetNCols(pRow) - 1; if (nCols <= 0) { pVal->valType = TD_VTYPE_NONE; return true; @@ -488,7 +488,7 @@ bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCe STSRow *pRow = pIter->pRow; SKvRowIdx *pKvIdx = NULL; bool colFound = false; - col_id_t kvNCols = tdRowGetNCols(pRow); + col_id_t kvNCols = tdRowGetNCols(pRow) - 1; void *pColIdx = TD_ROW_COL_IDX(pRow); while (*nIdx < kvNCols) { @@ -534,7 +534,7 @@ bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, return TSDB_CODE_SUCCESS; } - if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; return terrno; } @@ -814,7 +814,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP - TASSERT(colIdx < tdRowGetNCols(pRow)); + TASSERT(colIdx < tdRowGetNCols(pRow) - 1); if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { output->valType = TD_VTYPE_NONE; return terrno; @@ -1054,14 +1054,14 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { break; case TD_ROW_KV: #ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols - 1); + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps); #endif len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + pBuilder->nBoundBitmaps; // add TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); - TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols - 1); + TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); break; default: TASSERT(0); From dc42544e9ef21b19c283476db3b5c27e8979933c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 10 Aug 2022 20:45:36 +0800 Subject: [PATCH 89/96] refactor(sync): add leader, follower call back2 --- source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/inc/syncRespMgr.h | 7 ++++--- source/libs/sync/src/syncMain.c | 13 ++++++++++++- source/libs/sync/src/syncRespMgr.c | 25 +++++++++++++++++++------ 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 856870caba..3e247e5d79 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -163,6 +163,7 @@ typedef struct SSyncNode { bool changing; int64_t startTime; + int64_t leaderTime; int64_t lastReplicateTime; } SSyncNode; diff --git a/source/libs/sync/inc/syncRespMgr.h b/source/libs/sync/inc/syncRespMgr.h index e37c0bb625..28978af77e 100644 --- a/source/libs/sync/inc/syncRespMgr.h +++ b/source/libs/sync/inc/syncRespMgr.h @@ -32,9 +32,9 @@ typedef struct SRespStub { } SRespStub; typedef struct SSyncRespMgr { - SHashObj * pRespHash; + SHashObj *pRespHash; int64_t ttl; - void * data; + void *data; TdThreadMutex mutex; uint64_t seqNum; } SSyncRespMgr; @@ -46,7 +46,8 @@ int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index); int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub); int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub); void syncRespClean(SSyncRespMgr *pObj); -void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl); +void syncRespCleanRsp(SSyncRespMgr *pObj); +void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5f3ff3015c..9f9fdf4844 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1100,6 +1100,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { int64_t timeNow = taosGetTimestampMs(); pSyncNode->startTime = timeNow; + pSyncNode->leaderTime = timeNow; pSyncNode->lastReplicateTime = timeNow; syncNodeEventLog(pSyncNode, "sync open"); @@ -2015,6 +2016,8 @@ void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term) { } } +void syncNodeLeaderChangeRsp(SSyncNode* pSyncNode) { syncRespCleanRsp(pSyncNode->pSyncRespMgr); } + void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -2028,6 +2031,9 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // reset elect timer syncNodeResetElectTimer(pSyncNode); + // send rsp to client + syncNodeLeaderChangeRsp(pSyncNode); + // call back if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpBecomeFollowerCb != NULL) { pSyncNode->pFsm->FpBecomeFollowerCb(pSyncNode->pFsm); @@ -2068,6 +2074,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // /\ UNCHANGED <> // void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { + pSyncNode->leaderTime = taosGetTimestampMs(); + // reset restoreFinish pSyncNode->restoreFinish = false; @@ -2954,8 +2962,11 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, } ths->restoreFinish = true; + int64_t restoreDelay = taosGetTimestampMs() - ths->leaderTime; + char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "restore finish, index:%" PRId64, pEntry->index); + snprintf(eventLog, sizeof(eventLog), "restore finish, index:%ld, elapsed:%ld ms, ", pEntry->index, + restoreDelay); syncNodeEventLog(ths, eventLog); } } diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index da04b19e17..d7ed864180 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -108,13 +108,19 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu return 0; // get none object } -void syncRespClean(SSyncRespMgr *pObj) { +void syncRespCleanRsp(SSyncRespMgr *pObj) { taosThreadMutexLock(&(pObj->mutex)); - syncRespCleanByTTL(pObj, pObj->ttl); + syncRespCleanByTTL(pObj, -1, true); taosThreadMutexUnlock(&(pObj->mutex)); } -void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) { +void syncRespClean(SSyncRespMgr *pObj) { + taosThreadMutexLock(&(pObj->mutex)); + syncRespCleanByTTL(pObj, pObj->ttl, false); + taosThreadMutexUnlock(&(pObj->mutex)); +} + +void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { SRespStub *pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, NULL); int cnt = 0; int sum = 0; @@ -126,12 +132,12 @@ void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) { while (pStub) { size_t len; - void * key = taosHashGetKey(pStub, &len); + void *key = taosHashGetKey(pStub, &len); uint64_t *pSeqNum = (uint64_t *)key; sum++; int64_t nowMS = taosGetTimestampMs(); - if (nowMS - pStub->createTime > ttl) { + if (nowMS - pStub->createTime > ttl || -1 == ttl) { taosArrayPush(delIndexArray, pSeqNum); cnt++; @@ -148,7 +154,14 @@ void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) { pStub->rpcMsg.pCont = NULL; pStub->rpcMsg.contLen = 0; - pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &(pStub->rpcMsg), cbMeta); + + // TODO: and make rpcMsg body, call commit cb + // pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &(pStub->rpcMsg), cbMeta); + + pStub->rpcMsg.code = TSDB_CODE_SYN_NOT_LEADER; + if (pStub->rpcMsg.info.handle != NULL) { + tmsgSendRsp(&(pStub->rpcMsg)); + } } pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, pStub); From 1420524dd0f9737575f6f12f027fef15a3b5c43d Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 10 Aug 2022 20:57:37 +0800 Subject: [PATCH 90/96] Update 01-docker.md --- docs/zh/05-get-started/01-docker.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index 741f6dfeab..9a8552826d 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -2,6 +2,9 @@ sidebar_label: Docker title: 通过 Docker 快速体验 TDengine --- +:::info +如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. +::: 本节首先介绍如何通过 Docker 快速体验 TDengine，然后介绍如何在 Docker 环境下体验 TDengine 的写入和查询功能。 From 7f4b46e0031d749a0eeb1d226332c138f7c68b88 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 10 Aug 2022 21:17:38 +0800 Subject: [PATCH 91/96] feat(stream): session window trigger delete --- include/common/tmsg.h | 20 ++++++++- include/common/tmsgdef.h | 1 + source/common/src/tmsg.c | 39 ++++++++++++++++- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/mnode/impl/src/mndStream.c | 17 +++++--- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/sma/smaTimeRange.c | 7 +-- source/dnode/vnode/src/tq/tqSink.c | 47 +++++++++++++++++---- source/dnode/vnode/src/vnd/vnodeSvr.c | 21 +++++++++ 9 files changed, 134 insertions(+), 21 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d26e99f423..9fe01b0cf6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1368,8 +1368,8 @@ typedef struct { int32_t numOfCols; int64_t skey; int64_t ekey; - int64_t version; // for stream - TSKEY watermark;// for stream + int64_t version; // for stream + TSKEY watermark; // for stream char data[]; } SRetrieveTableRsp; @@ -3079,6 +3079,22 @@ typedef struct SDeleteRes { int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); int32_t tDecodeDeleteRes(SDecoder* pCoder, SDeleteRes* pRes); +typedef struct { + int64_t uid; + int64_t ts; +} SSingleDeleteReq; + +int32_t tEncodeSSingleDeleteReq(SEncoder* pCoder, const SSingleDeleteReq* pReq); +int32_t tDecodeSSingleDeleteReq(SDecoder* pCoder, SSingleDeleteReq* pReq); + +typedef struct { + int64_t suid; + SArray* deleteReqs; // SArray +} SBatchDeleteReq; + +int32_t tEncodeSBatchDeleteReq(SEncoder* pCoder, const SBatchDeleteReq* pReq); +int32_t tDecodeSBatchDeleteReq(SDecoder* pCoder, SBatchDeleteReq* pReq); + typedef struct { int32_t msgIdx; int32_t msgType; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 9ddb872007..6462c7afbf 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -202,6 +202,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) TD_DEF_MSG_TYPE(TDMT_VND_FETCH_RSMA, "vnode-fetch-rsma", SRSmaFetchMsg, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) + TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index b630419fc4..7dd3ce34c3 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -305,7 +305,7 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) taosArrayPush(desc.subDesc, &sDesc); } } - + ASSERT(desc.subPlanNum == taosArrayGetSize(desc.subDesc)); taosArrayPush(pReq->query->queryDesc, &desc); @@ -5770,3 +5770,40 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { } return 0; } + +int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) { + if (tEncodeI64(pEncoder, pReq->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->ts) < 0) return -1; + return 0; +} + +int32_t tDecodeSSingleDeleteReq(SDecoder *pDecoder, SSingleDeleteReq *pReq) { + if (tDecodeI64(pDecoder, &pReq->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->ts) < 0) return -1; + return 0; +} + +int32_t tEncodeSBatchDeleteReq(SEncoder *pEncoder, const SBatchDeleteReq *pReq) { + if (tEncodeI64(pEncoder, pReq->suid) < 0) return -1; + int32_t sz = taosArrayGetSize(pReq->deleteReqs); + if (tEncodeI32(pEncoder, sz) < 0) return -1; + for (int32_t i = 0; i < sz; i++) { + SSingleDeleteReq *pOneReq = taosArrayGet(pReq->deleteReqs, i); + if (tEncodeSSingleDeleteReq(pEncoder, pOneReq) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) { + if (tDecodeI64(pDecoder, &pReq->suid) < 0) return -1; + int32_t sz; + if (tDecodeI32(pDecoder, &sz) < 0) return -1; + pReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + if (pReq->deleteReqs == NULL) return -1; + for (int32_t i = 0; i < sz; i++) { + SSingleDeleteReq deleteReq; + if (tDecodeSSingleDeleteReq(pDecoder, &deleteReq) < 0) return -1; + taosArrayPush(pReq->deleteReqs, &deleteReq); + } + return 0; +} diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 1b799b1e5e..d13daffa08 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -364,6 +364,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_CHECK_ALTER_INFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b172b00fab..b4af39e467 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -943,12 +943,17 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false); - char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n)); - varDataSetLen(targetSTB, strlen(varDataVal(targetSTB))); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); + if (pStream->targetSTbName[0] == 0) { + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, NULL, true); + } else { + char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n)); + varDataSetLen(targetSTB, strlen(varDataVal(targetSTB))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); + } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7b22e061f0..ffc1966733 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -172,7 +172,7 @@ int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list); SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId); + const char* stbFullName, int32_t vgId, SBatchDeleteReq* pDeleteReq); // sma int32_t smaInit(); diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 7f69acc889..8da397f0c3 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -119,7 +119,7 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { SName stbFullName = {0}; tNameFromString(&stbFullName, pCfg->dstTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); SVCreateStbReq pReq = {0}; - pReq.name = (char*)tNameGetTableName(&stbFullName); + pReq.name = (char *)tNameGetTableName(&stbFullName); pReq.suid = pCfg->dstTbUid; pReq.schemaRow = pCfg->schemaRow; pReq.schemaTag = pCfg->schemaTag; @@ -200,8 +200,9 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { goto _err; } + SBatchDeleteReq deleteReq; SSubmitReq *pSubmitReq = tdBlockToSubmit((const SArray *)msg, pTsmaStat->pTSchema, true, pTsmaStat->pTSma->dstTbUid, - pTsmaStat->pTSma->dstTbName, pTsmaStat->pTSma->dstVgId); + pTsmaStat->pTSma->dstTbName, pTsmaStat->pTSma->dstVgId, &deleteReq); if (!pSubmitReq) { smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), @@ -230,4 +231,4 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { _err: tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 6b4d5a2f3e..2b3283a395 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -16,7 +16,7 @@ #include "tq.h" SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId) { + const char* stbFullName, int32_t vgId, SBatchDeleteReq* deleteReq) { SSubmitReq* ret = NULL; SArray* schemaReqs = NULL; SArray* schemaReqSz = NULL; @@ -33,10 +33,13 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo schemaReqSz = taosArrayInit(sz, sizeof(int32_t)); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - STagVal tagVal = { - .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .i64 = (int64_t)pDataBlock->info.groupId, + if (pDataBlock->info.type == STREAM_DELETE_DATA) { + // + } + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, }; STag* pTag = NULL; taosArrayClear(tagArray); @@ -176,17 +179,45 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo } void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { - const SArray* pRes = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; + const SArray* pRes = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + SBatchDeleteReq deleteReq = {0}; tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, (int32_t)pRes->size); ASSERT(pTask->tbSink.pTSchema); + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); SSubmitReq* pReq = tdBlockToSubmit(pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid, - pTask->tbSink.stbFullName, pVnode->config.vgId); + pTask->tbSink.stbFullName, pVnode->config.vgId, &deleteReq); tqDebug("vgId:%d, task %d convert blocks over, put into write-queue", TD_VID(pVnode), pTask->taskId); + int32_t code; + int32_t len; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* buf = taosMemoryCalloc(1, len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + + if (taosArrayGetSize(deleteReq.deleteReqs) != 0) { + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = buf, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + taosArrayDestroy(deleteReq.deleteReqs); + /*tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema);*/ // build write msg SRpcMsg msg = { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e4e1e608e5..1f13bde0c1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -29,6 +29,7 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; @@ -190,6 +191,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp case TDMT_VND_DELETE: if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; + case TDMT_VND_BATCH_DEL: + if (vnodeProcessBatchDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + break; /* TQ */ case TDMT_VND_MQ_VG_CHANGE: if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), @@ -1053,6 +1057,23 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void return 0; } +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SBatchDeleteReq deleteReq; + SDecoder decoder; + tDecoderInit(&decoder, pReq, len); + tDecodeSBatchDeleteReq(&decoder, &deleteReq); + + int32_t sz = taosArrayGetSize(deleteReq.deleteReqs); + for (int32_t i = 0; i < sz; i++) { + SSingleDeleteReq *pOneReq = taosArrayGet(deleteReq.deleteReqs, i); + int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, pOneReq->uid, pOneReq->ts, pOneReq->ts); + if (code) { + // TODO + } + } + return 0; +} + static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; From e03adb140af0ebf9c90510d911c79d1e8ca5bd1a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 10 Aug 2022 21:42:30 +0800 Subject: [PATCH 92/96] feat: remove temporary interface declarations (#15940) * add a few stmt_ interface declartion * add taos_load_table_info * add taos_set_config(() interface * add more interfaces * warn 'reset query cache' only since lack engine support * update taos-tools * feat: remove temporary interface declaration * fix: remove temporary interfaces --- include/client/taos.h | 13 ------ source/client/src/TSDBJNIConnector.c | 61 +--------------------------- source/client/src/clientMain.c | 15 ------- 3 files changed, 1 insertion(+), 88 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index b7df0e4d29..dd7266bd96 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -196,15 +196,6 @@ DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, vo DLL_EXPORT void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); DLL_EXPORT const void *taos_get_raw_block(TAOS_RES *res); -// Shuduo: temporary enable for app build -#if 1 -typedef void (*__taos_sub_fn_t)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); -DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, __taos_sub_fn_t fp, - void *param, int interval); -DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub); -DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); -#endif - DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); @@ -281,10 +272,6 @@ DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res); /* ------------------------------ TMQ END -------------------------------- */ -#if 1 // Shuduo: temporary enable for app build -typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); -#endif - typedef enum { TSDB_SRV_STATUS_UNAVAILABLE = 0, TSDB_SRV_STATUS_NETWORK_OK = 1, diff --git a/source/client/src/TSDBJNIConnector.c b/source/client/src/TSDBJNIConnector.c index 38bbec24ce..37041da695 100644 --- a/source/client/src/TSDBJNIConnector.c +++ b/source/client/src/TSDBJNIConnector.c @@ -611,65 +611,6 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeConnectionIm } } -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNIEnv *env, jobject jobj, jlong con, - jboolean restart, jstring jtopic, - jstring jsql, jint jinterval) { - jlong sub = 0; - TAOS *taos = (TAOS *)con; - char *topic = NULL; - char *sql = NULL; - - jniGetGlobalMethod(env); - jniDebug("jobj:%p, in TSDBJNIConnector_subscribeImp", jobj); - - if (jtopic != NULL) { - topic = (char *)(*env)->GetStringUTFChars(env, jtopic, NULL); - } - if (jsql != NULL) { - sql = (char *)(*env)->GetStringUTFChars(env, jsql, NULL); - } - - if (topic == NULL || sql == NULL) { - jniDebug("jobj:%p, invalid argument: topic or sql is NULL", jobj); - return sub; - } - - TAOS_SUB *tsub = taos_subscribe(taos, (int)restart, topic, sql, NULL, NULL, jinterval); - sub = (jlong)tsub; - - if (sub == 0) { - jniDebug("jobj:%p, failed to subscribe: topic:%s", jobj, topic); - } else { - jniDebug("jobj:%p, successfully subscribe: topic: %s", jobj, topic); - } - - (*env)->ReleaseStringUTFChars(env, jtopic, topic); - (*env)->ReleaseStringUTFChars(env, jsql, sql); - - return sub; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub) { - jniDebug("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%lld", jobj, sub); - jniGetGlobalMethod(env); - - TAOS_SUB *tsub = (TAOS_SUB *)sub; - TAOS_RES *res = taos_consume(tsub); - - if (res == NULL) { - jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub); - return 0l; - } - - return (jlong)res; -} - -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, - jboolean keepProgress) { - TAOS_SUB *tsub = (TAOS_SUB *)sub; - taos_unsubscribe(tsub, keepProgress); -} - JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTableSqlImp(JNIEnv *env, jobject jobj, jlong con, jbyteArray jsql) { TAOS *tscon = (TAOS *)con; @@ -1087,4 +1028,4 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsert return JNI_OUT_OF_MEMORY; } return (jlong)tres; -} \ No newline at end of file +} diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 0d41d35721..0ec724c6d0 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -939,21 +939,6 @@ const void *taos_get_raw_block(TAOS_RES *res) { return pRequest->body.resInfo.pData; } -TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, - void *param, int interval) { - // TODO - return NULL; -} - -TAOS_RES *taos_consume(TAOS_SUB *tsub) { - // TODO - return NULL; -} - -void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { - // TODO -} - int taos_load_table_info(TAOS *taos, const char *tableNameList) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; From 0aba651a08cc151167151e689383a5e7ce24e5f0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 10 Aug 2022 22:08:21 +0800 Subject: [PATCH 93/96] chore: update taosa and rust connector for3.0 (#15958) * chore: add libtaos-ws for 3.0 * chore: update taosws-rs * chore: add libtaosws to install/remove script * chore: update taosws-rs * chore: update taosws-rs * chore: update taos-tools, taosws-rs for 3.0 * fix: packaging/tools/make_install.sh for 3.0 * chore: update taos-tools * chore: fix release script for 3.0 * chore: update taosws-rs for 3.0 * chore: add taows-rs submodule for 3.0 * chore: update taosws-rs for 3.0 * fix: install script support taosws for 3.0 * fix: script error handle for 3.0 * chore: update taosws-rs for 3.0 fix segfault * chore: change container_build for websocket build * fix: install script for taosws * fix: . * chore: update taosws-rs for 3.0 * chore: update taosws-rs for 3.0 * chore: update tools/CMakeLists.txt to allow compile taosws-rw on any platform * chore: taosws 648cc62 for 3.0 * chore: update taosws 29424d5 for 3.0 * chore: update cmake/taosws_CMakeLists.txt.in with new repo/commit * chore: update taosa 3d21433 and rust connector for new raw block --- cmake/taosadapter_CMakeLists.txt.in | 2 +- cmake/taosws_CMakeLists.txt.in | 2 +- tools/CMakeLists.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index ee1e7d3b31..ed8216be91 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG ed6a160 + GIT_TAG 3d21433 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index c6d42b686c..506559a245 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 97c4bac + GIT_TAG 7a54d21 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1f066daabe..5751c347e3 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -112,7 +112,7 @@ ELSE () COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND - COMMAND wget -c https://github.com/upx/upx/releases/download/v3.96/upx-3.96-${PLATFORM_ARCH_STR}_linux.tar.xz -O $ENV{HOME}/upx.tar.xz && tar -xvJf $ENV{HOME}/upx.tar.xz -C $ENV{HOME}/ --strip-components 1 > /dev/null && $ENV{HOME}/upx taosadapter || : + COMMAND wget -nc https://github.com/upx/upx/releases/download/v3.96/upx-3.96-${PLATFORM_ARCH_STR}_linux.tar.xz -O $ENV{HOME}/upx.tar.xz && tar -xvJf $ENV{HOME}/upx.tar.xz -C $ENV{HOME}/ --strip-components 1 > /dev/null && $ENV{HOME}/upx taosadapter || : COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ From 8675580758c3ae36f7cc25b123ddac8903892d38 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 10 Aug 2022 23:00:38 +0800 Subject: [PATCH 94/96] docs: update connector for 3.0 (#15957) --- .../14-reference/03-connector/03-connector.mdx | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/zh/14-reference/03-connector/03-connector.mdx b/docs/zh/14-reference/03-connector/03-connector.mdx index 7a4a85276e..e18759e600 100644 --- a/docs/zh/14-reference/03-connector/03-connector.mdx +++ b/docs/zh/14-reference/03-connector/03-connector.mdx @@ -10,14 +10,14 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 目前 TDengine 的原生接口连接器可支持的平台包括：X64/X86/ARM64/ARM32/MIPS/Alpha 等硬件平台，以及 Linux/Win64/Win32 等开发环境。对照矩阵如下： -| **CPU** | **OS** | **JDBC** | **Python** | **Go** | **Node.js** | **C#** | **Rust** | C/C++ | +| **CPU** | **OS** | **Java** | **Python** | **Go** | **Node.js** | **C#** | **Rust** | C/C++ | | -------------- | --------- | -------- | ---------- | ------ | ----------- | ------ | -------- | ----- | | **X86 64bit** | **Linux** | ● | ● | ● | ● | ● | ● | ● | | **X86 64bit** | **Win64** | ● | ● | ● | ● | ● | ● | ● | | **X86 64bit** | **Win32** | ● | ● | ● | ● | ○ | ○ | ● | | **X86 32bit** | **Win32** | ○ | ○ | ○ | ○ | ○ | ○ | ● | | **ARM64** | **Linux** | ● | ● | ● | ● | ○ | ○ | ● | -| **ARM32** | **Linux** | ● | ● | ● | ● | ○ | ○ | ● | +| **ARM32** | **Linux** | ○ | ○ | ○ | ○ | ○ | ○ | ● | | **MIPS 龙芯** | **Linux** | ○ | ○ | ○ | ○ | ○ | ○ | ○ | | **Alpha 申威** | **Linux** | ○ | ○ | -- | -- | -- | -- | ○ | | **X86 海光** | **Linux** | ○ | ○ | ○ | -- | -- | -- | ○ | @@ -32,6 +32,7 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | **TDengine 版本** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | | --------------------- | -------- | ---------- | ------------ | ------------- | --------------- | -------- | +| **3.0.0.0 及以上** | 3.0.0 | 当前版本 | 3.0 分支 | 3.0.0 | 3.0.0 | 当前版本 | | **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | | **2.4.0.6 及以上** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | | **2.4.0.4 - 2.4.0.5** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | @@ -50,7 +51,7 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **连续查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **订阅功能** | 支持 | 支持 | 支持 | 支持 | 支持 | 暂不支持 | +| ** TMQ ** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **Schemaless** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | @@ -58,17 +59,17 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 由于不同编程语言数据库框架规范不同，并不意味着所有 C/C++ 接口都需要对应封装支持。 ::: -### 使用 REST 接口 +### 使用 http (REST 或 WebSocket) 接口 | **功能特性** | **Java** | **Python** | **Go** | **C#（暂不支持）** | **Node.js** | **Rust** | | ------------------------------ | -------- | ---------- | -------- | ------------------ | ----------- | -------- | | **连接管理** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | | **普通查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | | **连续查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | -| **参数绑定** | 不支持 | 不支持 | 不支持 | N/A | 不支持 | 不支持 | -| **订阅功能** | 不支持 | 不支持 | 不支持 | N/A | 不支持 | 不支持 | -| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | N/A | 暂不支持 | 暂不支持 | -| **批量拉取（基于 WebSocket）** | 支持 | 暂不支持 | 暂不支持 | N/A | 暂不支持 | 暂不支持 | +| **参数绑定** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 | +| ** TMQ ** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 | +| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 暂不支持 | +| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 暂不支持 | N/A | 不支持 | 支持 | | **DataFrame** | 不支持 | 支持 | 不支持 | N/A | 不支持 | 不支持 | :::warning From a5ef854e87e3f1051baeb5f415f3f904acd79976 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 11 Aug 2022 07:45:49 +0800 Subject: [PATCH 95/96] row: code optimization --- include/common/trow.h | 23 ++-- source/common/src/trow.c | 186 ++++++++++++++++++++--------- source/dnode/vnode/src/tq/tqRead.c | 2 +- 3 files changed, 138 insertions(+), 73 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index ad9f52390e..6cc9355144 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -195,12 +195,6 @@ static FORCE_INLINE void tdRowSetVal(SCellVal *pVal, uint8_t valType, void *val) pVal->valType = valType; pVal->val = val; } -/** - * @brief Primary TS column not included. - * - * @param pRow - * @return FORCE_INLINE - */ static FORCE_INLINE col_id_t tdRowGetNCols(STSRow *pRow) { return *(col_id_t *)TD_ROW_NCOLS(pRow); } static FORCE_INLINE void tdRowCpy(void *dst, const STSRow *pRow) { memcpy(dst, pRow, TD_ROW_LEN(pRow)); } static FORCE_INLINE const char *tdRowEnd(STSRow *pRow) { return (const char *)POINTER_SHIFT(pRow, TD_ROW_LEN(pRow)); } @@ -325,20 +319,17 @@ typedef struct { col_id_t kvIdx; // [0, nKvCols) } STSRowIter; -void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); -void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema); +void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema); +void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); +bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); +bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal); + int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow); -bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); -bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal); -bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal); -bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, - SCellVal *pVal); -bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal); -void tdSCellValPrint(SCellVal *pVal, int8_t colType); +bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag); #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_ROW_H_*/ +#endif /*_TD_COMMON_ROW_H_*/ \ No newline at end of file diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d81fbf4629..24032c8df2 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -32,9 +32,13 @@ const uint8_t tdVTypeByte[2][3] = {{ }; // declaration -static uint8_t tdGetBitmapByte(uint8_t byte); -static int32_t tdCompareColId(const void *arg1, const void *arg2); -static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2); +static uint8_t tdGetBitmapByte(uint8_t byte); +static bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); +static bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal); +static bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, + col_id_t colIdx, SCellVal *pVal); +static bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal); +static void tdSCellValPrint(SCellVal *pVal, int8_t colType); // implementation /** @@ -330,7 +334,6 @@ void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterReset(&iter, row); printf("%s >>>type:%d,sver:%d ", tag, (int32_t)TD_ROW_TYPE(row), (int32_t)TD_ROW_SVER(row)); - STColumn *cols = (STColumn *)&iter.pSchema->columns; while (true) { SCellVal sVal = {.valType = 255, NULL}; @@ -340,7 +343,6 @@ void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); tdSCellValPrint(&sVal, cols[iter.colIdx - 1].type); } - printf("\n"); } @@ -422,6 +424,16 @@ void tdSCellValPrint(SCellVal *pVal, int8_t colType) { } } +static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { + if (*(col_id_t *)key1 > ((SKvRowIdx *)key2)->colId) { + return 1; + } else if (*(col_id_t *)key1 < ((SKvRowIdx *)key2)->colId) { + return -1; + } else { + return 0; + } +} + bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); @@ -458,6 +470,39 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } +bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + return true; + } + + if (TD_IS_TP_ROW(pIter->pRow)) { + STColumn *pCol = NULL; + STSchema *pSchema = pIter->pSchema; + while (pIter->colIdx < pSchema->numOfCols) { + pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key + if (colId == pCol->colId) { + break; + } else if (pCol->colId < colId) { + ++pIter->colIdx; + continue; + } else { + return false; + } + } + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + ++pIter->colIdx; + } else if (TD_IS_KV_ROW(pIter->pRow)) { + return tdSTSRowIterGetKvVal(pIter, colId, &pIter->kvIdx, pVal); + } else { + pVal->valType = TD_VTYPE_NONE; + terrno = TSDB_CODE_INVALID_PARA; + if (COL_REACH_END(colId, pIter->maxColId)) return false; + } + return true; +} + bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { if (pIter->colIdx >= pIter->pSchema->numOfCols) { return false; @@ -473,9 +518,10 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { } if (TD_IS_TP_ROW(pIter->pRow)) { - tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); } else if (TD_IS_KV_ROW(pIter->pRow)) { - tdGetKvRowValOfColEx(pIter, pCol->colId, &pIter->kvIdx, pVal); + tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal); + ASSERT(0); } else { ASSERT(0); } @@ -484,45 +530,7 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { return true; } -bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal) { - STSRow *pRow = pIter->pRow; - SKvRowIdx *pKvIdx = NULL; - bool colFound = false; - col_id_t kvNCols = tdRowGetNCols(pRow) - 1; - void *pColIdx = TD_ROW_COL_IDX(pRow); - - while (*nIdx < kvNCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, *nIdx * sizeof(SKvRowIdx)); - if (pKvIdx->colId == colId) { - ++(*nIdx); - pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); - colFound = true; - break; - } else if (pKvIdx->colId > colId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - ++(*nIdx); - } - } - - if (!colFound) { - if (colId <= pIter->maxColId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - return false; - } - } - - if (tdGetBitmapValType(pIter->pBitmap, (*nIdx) - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { - pVal->valType = TD_VTYPE_NONE; - } - - return true; -} - -bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { +bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { STSRow *pRow = pIter->pRow; if (pRow->statis == 0) { pVal->valType = TD_VTYPE_NORM; @@ -550,14 +558,41 @@ bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, return true; } -static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { - if (*(col_id_t *)key1 > ((SKvRowIdx *)key2)->colId) { - return 1; - } else if (*(col_id_t *)key1 < ((SKvRowIdx *)key2)->colId) { - return -1; - } else { - return 0; +bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal) { + STSRow *pRow = pIter->pRow; + SKvRowIdx *pKvIdx = NULL; + bool colFound = false; + col_id_t kvNCols = tdRowGetNCols(pRow) - 1; + void *pColIdx = TD_ROW_COL_IDX(pRow); + while (*nIdx < kvNCols) { + pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, *nIdx * sizeof(SKvRowIdx)); + if (pKvIdx->colId == colId) { + ++(*nIdx); + pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); + colFound = true; + break; + } else if (pKvIdx->colId > colId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + ++(*nIdx); + } } + + if (!colFound) { + if (colId <= pIter->maxColId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + return false; + } + } + + if (tdGetBitmapValType(pIter->pBitmap, pIter->kvIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + pVal->valType = TD_VTYPE_NONE; + } + + return true; } int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { @@ -673,7 +708,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { return 0; } -static int32_t tdCompareColId(const void *arg1, const void *arg2) { +static FORCE_INLINE int32_t tdCompareColId(const void *arg1, const void *arg2) { int32_t colId = *(int32_t *)arg1; STColumn *pCol = (STColumn *)arg2; @@ -686,6 +721,45 @@ static int32_t tdCompareColId(const void *arg1, const void *arg2) { } } +bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + return true; + } + + STSRow *pRow = pIter->pRow; + int16_t colIdx = -1; + if (TD_IS_TP_ROW(pRow)) { + STSchema *pSchema = pIter->pSchema; + STColumn *pCol = + (STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ); + if (!pCol) { + pVal->valType = TD_VTYPE_NONE; + if (COL_REACH_END(colId, pIter->maxColId)) return false; + return true; + } +#ifdef TD_SUPPORT_BITMAP + colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); +#endif + tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); + } else if (TD_IS_KV_ROW(pRow)) { + SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), + compareKvRowColId, TD_EQ); +#ifdef TD_SUPPORT_BITMAP + if (pIdx) { + colIdx = POINTER_DISTANCE(pIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); + } +#endif + tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); + } else { + if (COL_REACH_END(colId, pIter->maxColId)) return false; + pVal->valType = TD_VTYPE_NONE; + } + + return true; +} + int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { TASSERT(0); @@ -1250,7 +1324,7 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { pIter->pRow = pRow; pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); pIter->offset = 0; - pIter->colIdx = 0; + pIter->colIdx = 0; // PRIMARYKEY_TIMESTAMP_COL_ID; pIter->kvIdx = 0; } @@ -1288,4 +1362,4 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value); } -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index ced98c8afa..5d7814a045 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -325,7 +325,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { for (int32_t i = 0; i < colActual; i++) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SCellVal sVal = {0}; - if (!tdSTSRowIterNext(&iter, &sVal)) { + if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; } if (colDataAppend(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { From 446b3291e76b91a5463cbb3d71b58ad8e35f6a9a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 11 Aug 2022 09:53:51 +0800 Subject: [PATCH 96/96] fix: remove obsolete code --- source/common/src/trow.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 24032c8df2..565498a47b 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -521,7 +521,6 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); } else if (TD_IS_KV_ROW(pIter->pRow)) { tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal); - ASSERT(0); } else { ASSERT(0); }