From a62a7cc41f709c829e7ebe8ef0bbe0f3807f1307 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 1 Aug 2022 21:05:44 +0800 Subject: [PATCH 001/165] enh: aggregate and window add field for merge data blocks --- include/libs/nodes/plannodes.h | 2 ++ source/libs/nodes/src/nodesCodeFuncs.c | 17 ++++++++-- source/libs/planner/src/planLogicCreater.c | 20 +++++++----- source/libs/planner/src/planPhysiCreater.c | 3 ++ source/libs/planner/src/planSpliter.c | 4 +++ source/libs/planner/test/planGroupByTest.cpp | 34 ++++++++++---------- 6 files changed, 53 insertions(+), 27 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 2f6bb603c1..6f25cc45fb 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -377,6 +377,7 @@ typedef struct SAggPhysiNode { SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function SNodeList* pGroupKeys; SNodeList* pAggFuncs; + bool mergeDataBlock; } SAggPhysiNode; typedef struct SDownstreamSourceNode { @@ -415,6 +416,7 @@ typedef struct SWinodwPhysiNode { int8_t igExpired; EOrder inputTsOrder; EOrder outputTsOrder; + bool mergeDataBlock; } SWinodwPhysiNode; typedef struct SIntervalPhysiNode { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 9d15b01acf..74659179a9 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1773,6 +1773,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { static const char* jkAggPhysiPlanExprs = "Exprs"; static const char* jkAggPhysiPlanGroupKeys = "GroupKeys"; static const char* jkAggPhysiPlanAggFuncs = "AggFuncs"; +static const char* jkAggPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) { const SAggPhysiNode* pNode = (const SAggPhysiNode*)pObj; @@ -1787,6 +1788,9 @@ static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkAggPhysiPlanAggFuncs, pNode->pAggFuncs); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanMergeDataBlock, pNode->mergeDataBlock); + } return code; } @@ -1804,6 +1808,9 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkAggPhysiPlanAggFuncs, &pNode->pAggFuncs); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkAggPhysiPlanMergeDataBlock, &pNode->mergeDataBlock); + } return code; } @@ -1936,8 +1943,9 @@ static const char* jkWindowPhysiPlanTsEnd = "TsEnd"; static const char* jkWindowPhysiPlanTriggerType = "TriggerType"; static const char* jkWindowPhysiPlanWatermark = "Watermark"; static const char* jkWindowPhysiPlanIgnoreExpired = "IgnoreExpired"; -static const char* jkWindowPhysiPlanInputTsOrder = "inputTsOrder"; +static const char* jkWindowPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkWindowPhysiPlanOutputTsOrder = "outputTsOrder"; +static const char* jkWindowPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; @@ -1970,6 +1978,9 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanOutputTsOrder, pNode->outputTsOrder); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkWindowPhysiPlanMergeDataBlock, pNode->mergeDataBlock); + } return code; } @@ -2005,6 +2016,9 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkWindowPhysiPlanOutputTsOrder, pNode->outputTsOrder, code); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkWindowPhysiPlanMergeDataBlock, &pNode->mergeDataBlock); + } return code; } @@ -2147,7 +2161,6 @@ static int32_t jsonToPhysiSessionWindowNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysiWindowNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkSessionWindowPhysiPlanGap, pNode->gap, code); - ; } return code; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b51624336b..a799b2feb6 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -468,6 +468,10 @@ static int32_t createGroupKeysFromPartKeys(SNodeList* pPartKeys, SNodeList** pOu return TSDB_CODE_SUCCESS; } +static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { + return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; +} + static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; @@ -480,7 +484,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->hasLastRow = pSelect->hasLastRowFunc; pAgg->hasTimeLineFunc = pSelect->hasTimeLineFunc; - pAgg->node.groupAction = GROUP_ACTION_SET; + pAgg->node.groupAction = getGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = pAgg->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; @@ -551,7 +555,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc; pIdfRowsFunc->isUniqueFunc = pSelect->hasUniqueFunc; pIdfRowsFunc->isTimeLineFunc = pSelect->hasTimeLineFunc; - pIdfRowsFunc->node.groupAction = GROUP_ACTION_KEEP; + pIdfRowsFunc->node.groupAction = getGroupAction(pCxt, pSelect); pIdfRowsFunc->node.requireDataOrder = pIdfRowsFunc->isTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; pIdfRowsFunc->node.resultDataOrder = pIdfRowsFunc->node.requireDataOrder; @@ -586,7 +590,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p return TSDB_CODE_OUT_OF_MEMORY; } - pInterpFunc->node.groupAction = GROUP_ACTION_KEEP; + pInterpFunc->node.groupAction = getGroupAction(pCxt, pSelect); pInterpFunc->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder; @@ -662,7 +666,7 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo } pWindow->winType = WINDOW_TYPE_STATE; - pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pWindow->pStateExpr = nodesCloneNode(pState->pExpr); @@ -685,7 +689,7 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW pWindow->winType = WINDOW_TYPE_SESSION; pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i; pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? SESSION_ALGO_STREAM_SINGLE : SESSION_ALGO_MERGE; - pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; @@ -714,7 +718,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; - pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); pWindow->node.requireDataOrder = pSelect->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_IN_BLOCK; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; @@ -762,7 +766,7 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return TSDB_CODE_OUT_OF_MEMORY; } - pFill->node.groupAction = GROUP_ACTION_KEEP; + pFill->node.groupAction = getGroupAction(pCxt, pSelect); pFill->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pFill->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pFill->inputTsOrder = ORDER_ASC; @@ -936,7 +940,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = GROUP_ACTION_SET; + pAgg->node.groupAction = GROUP_ACTION_CLEAR; pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 3771586b34..af3d4a9577 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -825,6 +825,8 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, return TSDB_CODE_OUT_OF_MEMORY; } + pAgg->mergeDataBlock = (GROUP_ACTION_KEEP == pAggLogicNode->node.groupAction ? false : true); + SNodeList* pPrecalcExprs = NULL; SNodeList* pGroupKeys = NULL; SNodeList* pAggFuncs = NULL; @@ -1091,6 +1093,7 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pWindow->igExpired = pWindowLogicNode->igExpired; pWindow->inputTsOrder = pWindowLogicNode->inputTsOrder; pWindow->outputTsOrder = pWindowLogicNode->outputTsOrder; + pWindow->mergeDataBlock = (GROUP_ACTION_KEEP == pWindowLogicNode->node.groupAction ? false : true); SNodeList* pPrecalcExprs = NULL; SNodeList* pFuncs = NULL; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 74c78b970d..6ddb6d90c0 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -393,6 +393,8 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic code = TSDB_CODE_OUT_OF_MEMORY; } + pPartWin->node.groupAction = GROUP_ACTION_KEEP; + if (TSDB_CODE_SUCCESS == code) { pMergeWindow->node.pTargets = pTargets; pMergeWindow->node.pConditions = pConditions; @@ -723,6 +725,8 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO code = TSDB_CODE_OUT_OF_MEMORY; } + pPartAgg->node.groupAction = GROUP_ACTION_KEEP; + if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { pPartAgg->pGroupKeys = pGroupKeys; code = createColumnByRewriteExprs(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets); diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index 657d8a6244..a553d3addc 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -32,6 +32,22 @@ TEST_F(PlanGroupByTest, basic) { run("SELECT c1 + c3, SUM(c4 * c5) FROM t1 WHERE CONCAT(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); run("SELECT SUM(CEIL(c1)) FROM t1 GROUP BY CEIL(c1)"); + + run("SELECT COUNT(*) FROM st1"); + + run("SELECT c1 FROM st1 GROUP BY c1"); + + run("SELECT COUNT(*) FROM st1 GROUP BY c1"); + + run("SELECT SUM(c1) FROM st1 GROUP BY c2 HAVING SUM(c1) IS NOT NULL"); +} + +TEST_F(PlanGroupByTest, withPartitionBy) { + useDb("root", "test"); + + run("SELECT LAST(ts), TBNAME FROM st1 PARTITION BY TBNAME"); + + run("SELECT COUNT(*) FROM st1 PARTITION BY c2 GROUP BY c1"); } TEST_F(PlanGroupByTest, withOrderBy) { @@ -43,14 +59,12 @@ TEST_F(PlanGroupByTest, withOrderBy) { // run("SELECT COUNT(*), SUM(c1) a FROM t1 ORDER BY a"); } -TEST_F(PlanGroupByTest, aggFunc) { +TEST_F(PlanGroupByTest, multiResFunc) { useDb("root", "test"); run("SELECT LAST(*), FIRST(*) FROM t1"); run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1"); - - run("SELECT SUM(10), COUNT(c1) FROM t1 GROUP BY c2"); } TEST_F(PlanGroupByTest, selectFunc) { @@ -67,17 +81,3 @@ TEST_F(PlanGroupByTest, selectFunc) { run("SELECT MAX(c1), c2 FROM t1 GROUP BY c3"); run("SELECT MAX(c1), t1.* FROM t1 GROUP BY c3"); } - -TEST_F(PlanGroupByTest, stable) { - useDb("root", "test"); - - run("SELECT COUNT(*) FROM st1"); - - run("SELECT c1 FROM st1 GROUP BY c1"); - - run("SELECT COUNT(*) FROM st1 GROUP BY c1"); - - run("SELECT COUNT(*) FROM st1 PARTITION BY c2 GROUP BY c1"); - - run("SELECT SUM(c1) FROM st1 GROUP BY c2 HAVING SUM(c1) IS NOT NULL"); -} From 781c494c2be7d2209eaa18ae99bbafeccf18af32 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 13:58:54 +0800 Subject: [PATCH 002/165] refactor: merge data belongs to different data group. --- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/executorimpl.c | 76 +++++++++---------- source/libs/executor/src/timewindowoperator.c | 1 + 3 files changed, 37 insertions(+), 44 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 577f9772be..e89459e555 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -519,6 +519,7 @@ typedef struct SBlockDistInfo { typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; SSDataBlock* pRes; + bool mergeResultBlock; } SOptrBasicInfo; typedef struct SIntervalAggOperatorInfo { @@ -593,7 +594,6 @@ typedef struct SAggOperatorInfo { uint64_t groupId; SGroupResInfo groupResInfo; SExprSupp scalarExprSup; - SNode *pCondition; } SAggOperatorInfo; @@ -891,7 +891,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, - int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); + int32_t numOfScalarExpr, bool mergeResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e52cbf40a9..a7b843d9c2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -136,9 +136,8 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, void operatorDummyCloseFn(void* param, int32_t numOfCols) {} -static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, - SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset, - SqlFunctionCtx* pCtx, int32_t numOfExprs); +static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); @@ -1501,19 +1500,24 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi return 0; } -int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, - SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, - int32_t numOfExprs) { - int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); - int32_t start = pGroupResInfo->index; +int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo) { + SExprInfo* pExprInfo = pSup->pExprInfo; + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; - for (int32_t i = start; i < numOfRows; i += 1) { + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + + for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); - doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset); + doUpdateNumOfRows(pRow, numOfExprs, rowEntryOffset); + + // no results, continue to check the next one if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; releaseBufPage(pBuf, page); @@ -1531,6 +1535,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); releaseBufPage(pBuf, page); break; } @@ -1540,7 +1545,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI for (int32_t j = 0; j < numOfExprs; ++j) { int32_t slotId = pExprInfo[j].base.resSchema.slotId; - pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowCellOffset); + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (pCtx[j].fpSet.finalize) { #ifdef BUF_PAGE_DEBUG qDebug("\npage_finalize %d", numOfExprs); @@ -1573,26 +1578,19 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI releaseBufPage(pBuf, page); pBlock->info.rows += pRow->numOfRows; - // if (pBlock->info.rows >= pBlock->info.capacity) { // output buffer is full - // break; - // } } qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.groupId); + blockDataUpdateTsWindow(pBlock, 0); return 0; } void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { - SExprInfo* pExprInfo = pOperator->exprSupp.pExprInfo; - int32_t numOfExprs = pOperator->exprSupp.numOfExprs; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int32_t* rowCellOffset = pOperator->exprSupp.rowEntryInfoOffset; - SSDataBlock* pBlock = pbInfo->pRes; - SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; + SSDataBlock* pBlock = pbInfo->pRes; // set output datablock version pBlock->info.version = pTaskInfo->version; @@ -1604,30 +1602,22 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG // clear the existed group id pBlock->info.groupId = 0; - doCopyToSDataBlock(pTaskInfo, pBlock, pExprInfo, pBuf, pGroupResInfo, rowCellOffset, pCtx, numOfExprs); -} - -static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, - int32_t* rowEntryInfoOffset) { - // update the number of result for each, only update the number of rows for the corresponding window result. - // if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - // return; - // } -#if 0 - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow* pResult = pResultRowInfo->pResult[i]; - - for (int32_t j = 0; j < numOfOutput; ++j) { - int32_t functionId = pCtx[j].functionId; - if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ) { - continue; + if (!pbInfo->mergeResultBlock) { + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo); + } else { + while(hasRemainResults(pGroupResInfo)) { + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo); + if (pBlock->info.rows >= pOperator->resultInfo.threshold) { + break; } - SResultRowEntryInfo* pCell = getResultEntryInfo(pResult, j, rowEntryInfoOffset); - pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); + // clearing group id to continue to merge data that belong to different groups + pBlock->info.groupId = 0; } + + // clear the group id info in SSDataBlock, since the client does not need it + pBlock->info.groupId = 0; } -#endif } static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, int8_t compressed) { @@ -2979,6 +2969,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { break; } } + size_t rows = blockDataGetNumOfRows(pInfo->pRes); pOperator->resultInfo.totalRows += rows; @@ -3483,7 +3474,7 @@ void cleanupExprSupp(SExprSupp* pSupp) { SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, - int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo) { + int32_t numOfScalarExpr, bool mergeResult, SExecTaskInfo* pTaskInfo) { SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3505,6 +3496,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } + pInfo->binfo.mergeResultBlock = mergeResult; pInfo->groupId = UINT64_MAX; pInfo->pCondition = pCondition; pOperator->name = "TableAggregate"; @@ -4098,7 +4090,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pScalarExprInfo, numOfScalarExpr, pTaskInfo); } else { pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, - pScalarExprInfo, numOfScalarExpr, pTaskInfo); + pScalarExprInfo, numOfScalarExpr, pAggNode->mergeDataBlock, pTaskInfo); } } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 802e1f2306..c447d6b43b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1798,6 +1798,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->twAggSup = *pTwAggSupp; pInfo->ignoreExpiredData = pPhyNode->window.igExpired; pInfo->pCondition = pPhyNode->window.node.pConditions; + pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock; if (pPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; From 770cc74e7f220b2465b3c0402f2c2bd9844c51c6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 14:19:15 +0800 Subject: [PATCH 003/165] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8821dbd5a1..43210ced2a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -464,6 +464,11 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pBlock->info = binfo; ASSERT(binfo.uid != 0); + uint64_t* groupId = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); + if (groupId) { + pBlock->info.groupId = *groupId; + } + uint32_t status = 0; int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); @@ -476,11 +481,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - uint64_t* groupId = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); - if (groupId) { - pBlock->info.groupId = *groupId; - } - pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows; pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; From be51233c241d906c9525d38e95c1943fb7a77413 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 14:26:54 +0800 Subject: [PATCH 004/165] refactor: add dynamic prune for last partial. --- source/libs/function/src/builtins.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6383179fee..23e26f7716 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2343,6 +2343,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_LAST_PARTIAL, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLastPartial, + .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastFunction, From cbed4713afccd050c7d4189dd460b9517eb993d9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 14:34:50 +0800 Subject: [PATCH 005/165] refactor: do some internal refactor. --- source/libs/function/src/functionMgt.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 339ff3808b..72ba81c3a7 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -116,6 +116,11 @@ EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, STimeWindow* return TSDB_CODE_FAILED; } + const char* name = funcMgtBuiltins[funcId].name; + if ((strcmp(name, "_group_key") == 0) || (strcmp(name, "_select_value") == 0)) { + return FUNC_DATA_REQUIRED_NOT_LOAD; + } + if (funcMgtBuiltins[funcId].dynDataRequiredFunc == NULL) { return FUNC_DATA_REQUIRED_DATA_LOAD; } else { From 258d50306fa5295fa21bb9434bdcdfcd7f8d0e71 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 15:43:45 +0800 Subject: [PATCH 006/165] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/cachescanoperator.c | 2 +- source/libs/executor/src/executorimpl.c | 5 +- source/libs/executor/src/timewindowoperator.c | 124 +++++++++--------- 4 files changed, 63 insertions(+), 72 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index e89459e555..d24156bc60 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -554,8 +554,6 @@ typedef struct SMergeAlignedIntervalAggOperatorInfo { uint64_t groupId; // current groupId int64_t curTs; // current ts SSDataBlock* prefetchedBlock; - bool inputBlocksFinished; - SNode* pCondition; } SMergeAlignedIntervalAggOperatorInfo; @@ -906,7 +904,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - SExecTaskInfo* pTaskInfo); + bool mergeResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 56a5e253d8..94e4384b30 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -50,7 +50,7 @@ SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pScanNode, SRead STableListInfo* pTableList = &pTaskInfo->tableqinfoList; - initResultSizeInfo(&pOperator->resultInfo, 1024); + initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); pInfo->pUidList = taosArrayInit(4, sizeof(int64_t)); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a7b843d9c2..794e751dd2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3481,10 +3481,9 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - int32_t numOfRows = 1024; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, numOfRows); + initResultSizeInfo(&pOperator->resultInfo, 4096); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -4147,7 +4146,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; - pOptr = createMergeIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pTaskInfo); + pOptr = createMergeIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pIntervalPhyNode->window.mergeDataBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL == type) { int32_t children = 0; pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c447d6b43b..812428920a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4622,20 +4622,25 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR int32_t numOfOutput = pSup->numOfExprs; int64_t* tsCols = extractTsCol(pBlock, iaInfo); uint64_t tableGroupId = pBlock->info.groupId; - TSKEY currTs = getStartTsKey(&pBlock->info.window, tsCols); SResultRow* pResult = NULL; + TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); + // there is an result exists if (miaInfo->curTs != INT64_MIN) { ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); - if (currTs != miaInfo->curTs) { + + if (ts != miaInfo->curTs) { outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs); - miaInfo->curTs = INT64_MIN; + miaInfo->curTs = ts; } + } else { + miaInfo->curTs = ts; + ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); } STimeWindow win = {0}; - win.skey = currTs; + win.skey = miaInfo->curTs; win.ekey = taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1; @@ -4646,12 +4651,11 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - miaInfo->curTs = win.skey; int32_t currPos = startPos; STimeWindow currWin = win; while (++currPos < pBlock->info.rows) { - if (tsCols[currPos] == currTs) { + if (tsCols[currPos] == miaInfo->curTs) { continue; } @@ -4659,11 +4663,10 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder); - outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs); - miaInfo->curTs = INT64_MIN; + outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs); + miaInfo->curTs = tsCols[currPos]; - currTs = tsCols[currPos]; - currWin.skey = currTs; + currWin.skey = miaInfo->curTs; currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1; @@ -4680,14 +4683,6 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder); - - if (currPos >= pBlock->info.rows) { - // we need to see next block if exists - } else { - ASSERT(0); - outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs); - miaInfo->curTs = INT64_MIN; - } } static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { @@ -4703,59 +4698,56 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SSDataBlock* pRes = iaInfo->binfo.pRes; blockDataCleanup(pRes); - blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); - if (!miaInfo->inputBlocksFinished) { - SOperatorInfo* downstream = pOperator->pDownstream[0]; - int32_t scanFlag = MAIN_SCAN; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + int32_t scanFlag = MAIN_SCAN; - while (1) { - SSDataBlock* pBlock = NULL; - if (miaInfo->prefetchedBlock == NULL) { - pBlock = downstream->fpSet.getNextFn(downstream); - } else { - pBlock = miaInfo->prefetchedBlock; - miaInfo->groupId = pBlock->info.groupId; - miaInfo->prefetchedBlock = NULL; - } + while (1) { + SSDataBlock* pBlock = NULL; + if (miaInfo->prefetchedBlock == NULL) { + pBlock = downstream->fpSet.getNextFn(downstream); + } else { + pBlock = miaInfo->prefetchedBlock; + miaInfo->prefetchedBlock = NULL; - if (pBlock == NULL) { - // close last unfinalized time window - if (miaInfo->curTs != INT64_MIN) { - ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); - outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); - miaInfo->curTs = INT64_MIN; - } - - doSetOperatorCompleted(pOperator); - miaInfo->inputBlocksFinished = true; - break; - } - - if (!miaInfo->hasGroupId) { - miaInfo->hasGroupId = true; - miaInfo->groupId = pBlock->info.groupId; - } else if (miaInfo->groupId != pBlock->info.groupId) { - // if there are unclosed time window, close it firstly. - ASSERT(miaInfo->curTs != INT64_MIN); - outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); - miaInfo->prefetchedBlock = pBlock; - miaInfo->curTs = INT64_MIN; - break; - } - - getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag); - setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true); - doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); - doFilter(miaInfo->pCondition, pRes, NULL); - if (pRes->info.rows >= pOperator->resultInfo.capacity) { - break; - } + miaInfo->groupId = pBlock->info.groupId; } - pRes->info.groupId = miaInfo->groupId; + if (pBlock == NULL) { + // close last unfinalized time window + if (miaInfo->curTs != INT64_MIN) { + ASSERT(taosHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); + outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); + miaInfo->curTs = INT64_MIN; + } + + doSetOperatorCompleted(pOperator); + break; + } + + if (!miaInfo->hasGroupId) { + miaInfo->hasGroupId = true; + miaInfo->groupId = pBlock->info.groupId; + } else if (miaInfo->groupId != pBlock->info.groupId) { + // if there are unclosed time window, close it firstly. + ASSERT(miaInfo->curTs != INT64_MIN); + outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); + miaInfo->prefetchedBlock = pBlock; + miaInfo->curTs = INT64_MIN; + break; + } + + getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag); + setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true); + doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); + + doFilter(miaInfo->pCondition, pRes, NULL); + if (pRes->info.rows >= pOperator->resultInfo.capacity) { + break; + } } + pRes->info.groupId = miaInfo->groupId; miaInfo->hasGroupId = false; size_t rows = pRes->info.rows; @@ -4809,6 +4801,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, } initResultRowInfo(&iaInfo->binfo.resultRowInfo); + blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity); pOperator->name = "TimeMergeAlignedIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL; @@ -5076,7 +5069,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - SExecTaskInfo* pTaskInfo) { + bool mergeBlock, SExecTaskInfo* pTaskInfo) { SMergeIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (miaInfo == NULL || pOperator == NULL) { @@ -5090,6 +5083,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI iaInfo->inputOrder = TSDB_ORDER_ASC; iaInfo->interval = *pInterval; iaInfo->execModel = pTaskInfo->execModel; + iaInfo->binfo.mergeResultBlock = mergeBlock; iaInfo->primaryTsIndex = primaryTsSlotId; From 70284f5b0202317b27c05f961ee476f2ba406d8a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 16:52:36 +0800 Subject: [PATCH 007/165] enh(query): support multi-groups result merge into one datablock. --- source/libs/executor/src/timewindowoperator.c | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 812428920a..5dcfb400d9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4685,20 +4685,16 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR tsCols, pBlock->info.rows, numOfOutput, iaInfo->inputOrder); } -static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { + +static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperator->info; SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } SExprSupp* pSup = &pOperator->exprSupp; SSDataBlock* pRes = iaInfo->binfo.pRes; - blockDataCleanup(pRes); - SOperatorInfo* downstream = pOperator->pDownstream[0]; int32_t scanFlag = MAIN_SCAN; @@ -4749,6 +4745,35 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { pRes->info.groupId = miaInfo->groupId; miaInfo->hasGroupId = false; +} + +static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info; + SIntervalAggOperatorInfo* iaInfo = pMiaInfo->intervalAggOperatorInfo; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSDataBlock* pRes = iaInfo->binfo.pRes; + blockDataCleanup(pRes); + + if (iaInfo->binfo.mergeResultBlock) { + while(1) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pRes->info.rows >= pOperator->resultInfo.threshold) { + break; + } + + doMergeAlignedIntervalAgg(pOperator); + } + } else { + doMergeAlignedIntervalAgg(pOperator); + } size_t rows = pRes->info.rows; pOperator->resultInfo.totalRows += rows; @@ -4812,7 +4837,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->exprSupp.numOfExprs = numOfCols; pOperator->info = miaInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeAlignedIntervalAgg, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL, destroyMergeAlignedIntervalOperatorInfo, NULL, NULL, NULL); code = appendDownstream(pOperator, &downstream, 1); From f98945e111e28061ee3c22d3d0f497262ffe9ff9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 2 Aug 2022 18:57:55 +0800 Subject: [PATCH 008/165] change log level --- source/libs/transport/src/transCli.c | 4 ++-- source/libs/transport/src/transSvr.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 431e479123..7ab3102d1c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -381,7 +381,7 @@ void cliHandleResp(SCliConn* conn) { STraceId* trace = &transMsg.info.traceId; - tGTrace("%s conn %p %s received from %s, local info:%s, len:%d, code str:%s", CONN_GET_INST_LABEL(conn), conn, + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, code str:%s", CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(pHead->msgType), conn->dst, conn->src, transMsg.contLen, tstrerror(transMsg.code)); if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { @@ -757,7 +757,7 @@ void cliSend(SCliConn* pConn) { uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); STraceId* trace = &pMsg->info.traceId; - tGTrace("%s conn %p %s is sent to %s, local info %s, len:%d", CONN_GET_INST_LABEL(pConn), pConn, + tGDebug("%s conn %p %s is sent to %s, local info %s, len:%d", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, pMsg->contLen); if (pHead->persist == 1) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 8b27d95e52..ea9d7bb910 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -253,10 +253,10 @@ static void uvHandleReq(SSvrConn* pConn) { if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); - tGTrace("%s conn %p %s received from %s, local info:%s, len:%d", transLabel(pTransInst), pConn, + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, transMsg.contLen); } else { - tGTrace("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d", transLabel(pTransInst), pConn, + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, transMsg.contLen, pHead->noResp, transMsg.code); } @@ -416,7 +416,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { STrans* pTransInst = pConn->pTransInst; STraceId* trace = &pMsg->info.traceId; - tGTrace("%s conn %p %s is sent to %s, local info:%s, len:%d", transLabel(pTransInst), pConn, + tGDebug("%s conn %p %s is sent to %s, local info:%s, len:%d", transLabel(pTransInst), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, pMsg->contLen); pHead->msgLen = htonl(len); From 77ed2bc1e13056f47c1db7f64d800eec68aa6abd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Aug 2022 21:25:55 +0800 Subject: [PATCH 009/165] enh(query): add merge attribute in explain. --- source/libs/command/inc/commandInt.h | 3 ++- source/libs/command/src/explain.c | 18 +++++++++++++++--- source/libs/executor/inc/executorimpl.h | 6 +++--- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/timewindowoperator.c | 5 +++-- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 2ae4666ff6..c53d464eab 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -49,6 +49,7 @@ extern "C" { #define EXPLAIN_PARITION_FORMAT "Partition on Column %s" #define EXPLAIN_ORDER_FORMAT "Order: %s" #define EXPLAIN_FILTER_FORMAT "Filter: " +#define EXPLAIN_MERGEBLOCKS_FORMAT "Merge ResBlocks: %s" #define EXPLAIN_FILL_VALUE_FORMAT "Fill Values: " #define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " #define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" @@ -56,7 +57,7 @@ extern "C" { #define EXPLAIN_TIME_WINDOWS_FORMAT "Time Window: interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c" #define EXPLAIN_WINDOW_FORMAT "Window: gap=%" PRId64 #define EXPLAIN_RATIO_TIME_FORMAT "Ratio: %f" -#define EXPLAIN_MERGE_FORMAT "Merge" +#define EXPLAIN_MERGE_FORMAT "SortMerge" #define EXPLAIN_MERGE_KEYS_FORMAT "Merge Key: " #define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms" diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index c080b666cc..2836bd7acd 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -702,7 +702,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pAggNode->mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } @@ -901,7 +905,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pIntNode->window.mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } @@ -945,7 +953,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pIntNode->window.mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d24156bc60..6383ba06b3 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -908,10 +908,10 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - SNode* pCondition, SExecTaskInfo* pTaskInfo); + SNode* pCondition, bool mergeResultBlocks, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); +SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, int32_t numOfChild); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 794e751dd2..112b6ebc99 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4131,7 +4131,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, - pPhyNode->pConditions, pTaskInfo); + pPhyNode->pConditions, pIntervalPhyNode->window.mergeDataBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) { SMergeIntervalPhysiNode* pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5dcfb400d9..aaa7a1ea8d 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4762,7 +4762,7 @@ static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) { if (iaInfo->binfo.mergeResultBlock) { while(1) { if (pOperator->status == OP_EXEC_DONE) { - return NULL; + break; } if (pRes->info.rows >= pOperator->resultInfo.threshold) { @@ -4782,7 +4782,7 @@ static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, - int32_t primaryTsSlotId, SNode* pCondition, + int32_t primaryTsSlotId, SNode* pCondition, bool mergeResultBlock, SExecTaskInfo* pTaskInfo) { SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -4806,6 +4806,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, iaInfo->interval = *pInterval; iaInfo->execModel = pTaskInfo->execModel; iaInfo->primaryTsIndex = primaryTsSlotId; + iaInfo->binfo.mergeResultBlock = mergeResultBlock; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(&pOperator->resultInfo, 4096); From 64db9afeb7c49d063d8fb6801d356ad2428212c6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 10:52:06 +0800 Subject: [PATCH 010/165] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 84 +++++++++++++++----------- 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c003f5a63f..6d1e3a4e07 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1205,6 +1205,26 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* return code; } +static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, int64_t key, SFileBlockDumpInfo* pDumpInfo) { + + // opt version + // 1. it is not a border point + // 2. the direct next point is not an duplicated timestamp + if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && pReader->order == TSDB_ORDER_ASC) || + (pDumpInfo->rowIndex > 0 && pReader->order == TSDB_ORDER_DESC)) { + int32_t step = pReader->order == TSDB_ORDER_ASC? 1:-1; + + int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; + if (nextKey != key) { // merge is not needed + doAppendRowFromBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + pDumpInfo->rowIndex += step; + return true; + } + } + + return false; +} + static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key) { SRowMerger merge = {0}; @@ -1219,10 +1239,14 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* // ascending order traverse if (ASCENDING_TRAVERSE(pReader->order)) { if (key < k.ts) { - tRowMergerInit(&merge, &fRow, pReader->pSchema); - - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); + // imem & mem are all empty, only file exist + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + } } else if (k.ts < key) { // k.ts < key doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); } else { // k.ts == key, ascending order: file block ----> imem rows -----> mem rows @@ -1238,10 +1262,13 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (key < k.ts) { doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); } else if (k.ts < key) { - tRowMergerInit(&merge, &fRow, pReader->pSchema); - - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + } } else { // descending order: mem rows -----> imem rows ------> file block updateSchema(pRow, pBlockScanInfo->uid, pReader); @@ -1438,34 +1465,23 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } // imem & mem are all empty, only file exist + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - // opt version - // 1. it is not a border point - // 2. the direct next point is not an duplicated timestamp - if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && pReader->order == TSDB_ORDER_ASC) || - (pDumpInfo->rowIndex > 0 && pReader->order == TSDB_ORDER_DESC)) { - int32_t step = pReader->order == TSDB_ORDER_ASC? 1:-1; - int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; - if (nextKey != key) { // merge is not needed - doAppendRowFromBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); - pDumpInfo->rowIndex += step; - return TSDB_CODE_SUCCESS; - } + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; } - - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - - STSRow* pTSRow = NULL; - SRowMerger merge = {0}; - - tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); - - taosMemoryFree(pTSRow); - tRowMergerClear(&merge); - return TSDB_CODE_SUCCESS; } } From bc600a4942e5a82cad3d2749f613838f9315b33a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 14:08:03 +0800 Subject: [PATCH 011/165] refactor: do some internal refactor optimize the building block performance. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 76 +++++++++++++++++++------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6d1e3a4e07..62d7a72b8f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -152,7 +152,7 @@ static void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); static void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, - STsdbReader* pReader); + STsdbReader* pReader, bool* freeTSRow); static void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow); static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData, @@ -1235,6 +1235,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* TSDBKEY k = TSDBROW_KEY(pRow); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SArray* pDelList = pBlockScanInfo->delSkyline; + bool freeTSRow = false; // ascending order traverse if (ASCENDING_TRAVERSE(pReader->order)) { @@ -1248,7 +1249,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerGetRow(&merge, &pTSRow); } } else if (k.ts < key) { // k.ts < key - doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader, &freeTSRow); } else { // k.ts == key, ascending order: file block ----> imem rows -----> mem rows tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); @@ -1260,7 +1261,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } else { // descending order scan if (key < k.ts) { - doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader, &freeTSRow); } else if (k.ts < key) { if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { return TSDB_CODE_SUCCESS; @@ -1302,12 +1303,12 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* ASSERT(pRow != NULL && piRow != NULL); int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + bool freeTSRow = false; uint64_t uid = pBlockScanInfo->uid; TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); - if (ASCENDING_TRAVERSE(pReader->order)) { // [1&2] key <= [k.ts && ik.ts] if (key <= k.ts && key <= ik.ts) { @@ -1335,16 +1336,22 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [3] ik.ts < key <= k.ts // [4] ik.ts < k.ts <= key if (ik.ts < k.ts) { - doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader); + doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } return TSDB_CODE_SUCCESS; } // [5] k.ts < key <= ik.ts // [6] k.ts < ik.ts <= key if (k.ts < ik.ts) { - doMergeMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader); + doMergeMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader, &freeTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } return TSDB_CODE_SUCCESS; } @@ -1354,6 +1361,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } } @@ -1385,8 +1393,11 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [3] ik.ts > k.ts >= Key // [4] ik.ts > key >= k.ts if (ik.ts > key) { - doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader); + doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } return TSDB_CODE_SUCCESS; } @@ -1399,18 +1410,21 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } //[7] key = ik.ts > k.ts if (key == ik.ts) { - doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader); + doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); tRowMerge(&merge, &fRow); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + + taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } } @@ -2280,11 +2294,29 @@ void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader) { } void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, - STsdbReader* pReader) { - SRowMerger merge = {0}; + STsdbReader* pReader, bool* freeTSRow) { + { // if the timestamp of the next valid row has a different ts, return current row directly + TSDBROW current = *pRow; + pIter->hasVal = tsdbTbDataIterNext(pIter->iter); - TSDBKEY k = TSDBROW_KEY(pRow); - // updateSchema(pRow, uid, pReader); + if (!pIter->hasVal) { + *pTSRow = current.pTSRow; + *freeTSRow = false; + return; + } else { // has next point in mem/imem + TSDBROW* pNextRow = getValidRow(pIter, pDelList, pReader); + if (TSDBROW_KEY(¤t).ts != TSDBROW_KEY(pNextRow).ts) { + *pTSRow = current.pTSRow; + *freeTSRow = false; + return; + } + } + } + + SRowMerger merge = {0}; + TSDBKEY k = TSDBROW_KEY(pRow); + + // get the correct schema for data in memory int32_t sversion = TSDBROW_SVERSION(pRow); STSchema* pTSchema = NULL; if (sversion != pReader->pSchema->version) { @@ -2298,6 +2330,8 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe tRowMergerGetRow(&merge, pTSRow); tRowMergerClear(&merge); + *freeTSRow = true; + if (sversion != pReader->pSchema->version) { taosMemoryFree(pTSchema); } @@ -2332,7 +2366,7 @@ void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlo } int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow, - int64_t endKey) { + int64_t endKey, bool* freeTSRow) { TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); SArray* pDelList = pBlockScanInfo->delSkyline; @@ -2358,23 +2392,24 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR TSDBKEY ik = TSDBROW_KEY(piRow); if (ik.ts < k.ts) { // ik.ts < k.ts - doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader); + doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); } else if (k.ts < ik.ts) { - doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); } else { // ik.ts == k.ts doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, pTSRow); + *freeTSRow = true; } return TSDB_CODE_SUCCESS; } if (pBlockScanInfo->iter.hasVal && pRow != NULL) { - doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader); + doMergeMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); return TSDB_CODE_SUCCESS; } if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { - doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader); + doMergeMultiRows(piRow, pBlockScanInfo->uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); return TSDB_CODE_SUCCESS; } @@ -2472,13 +2507,16 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e do { STSRow* pTSRow = NULL; - tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow, endKey); + bool freeTSRow = false; + tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow, endKey, &freeTSRow); if (pTSRow == NULL) { break; } doAppendRowFromTSRow(pBlock, pReader, pTSRow); - taosMemoryFree(pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } // no data in buffer, return immediately if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) { From 074500805c1745035d50217758115067360db5ae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 15:34:42 +0800 Subject: [PATCH 012/165] refactor: set filter function for last_row function. --- source/libs/executor/src/scanoperator.c | 5 ++--- source/libs/function/src/builtins.c | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 43210ced2a..b0f05e4705 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -521,7 +521,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; - qDebug("%s start to repeat ascending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); + qDebug("%s start to repeat ascending order scan data SELECT last_row(*),hostname from cpu group by hostname;blocks due to query func required", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); @@ -533,10 +533,9 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(pTableScanInfo, pOperator->exprSupp.pCtx, 0); tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); + qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); } - qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); - while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 23e26f7716..444cd2c5d0 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2271,6 +2271,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_LAST_ROW, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLast, + .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastRowFunction, From a3166507a54baf86a31d45f91935deecc9911229 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 15:58:28 +0800 Subject: [PATCH 013/165] enh: don't read bitmap when all col is normal --- include/common/trow.h | 8 ++-- source/client/src/tmq.c | 2 + source/common/src/tdatablock.c | 1 + source/common/src/trow.c | 56 +++++++++++++++---------- source/dnode/vnode/src/tq/tqSink.c | 1 + source/libs/executor/src/dataInserter.c | 1 + source/libs/parser/src/parInsert.c | 7 +++- source/libs/parser/src/parInsertData.c | 1 + 8 files changed, 52 insertions(+), 25 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 807a4c0f0a..93004b2cbd 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -38,7 +38,8 @@ typedef struct { uint16_t type : 2; uint16_t del : 1; uint16_t endian : 1; - uint16_t reserve : 12; + uint16_t normal : 1; // all norm + uint16_t reserve : 11; uint16_t sver; }; }; @@ -149,6 +150,8 @@ typedef struct { void *pBitmap; void *pOffset; int32_t extendedRowSize; + int16_t nNone; + int16_t nNull; } SRowBuilder; #define TD_ROW_HEAD_LEN (sizeof(STSRow)) @@ -287,9 +290,8 @@ int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen); int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, int32_t allNullLen, int32_t boundNullLen); int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); +int32_t tdSRowEnd(SRowBuilder *pBuilder); int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf); -int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, - int32_t nBoundCols, int32_t flen); void tdSRowReset(SRowBuilder *pBuilder); int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index bdd8c75f26..46349a3327 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -3129,6 +3129,7 @@ int taos_write_raw_block(TAOS *taos, int rows, char *pData, const char* tbname){ offset += TYPE_BYTES[pColumn->type]; } + tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); rowData = POINTER_SHIFT(rowData, rowLen); dataLen += rowLen; @@ -3360,6 +3361,7 @@ static int32_t tmqWriteRaw(TAOS *taos, void* data, int32_t dataLen){ } offset += TYPE_BYTES[pColumn->type]; } + tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); rowData = POINTER_SHIFT(rowData, rowLen); dataLen += rowLen; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 613fdd6b77..a16364b4a9 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1906,6 +1906,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks } offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation } + tdSRowEnd(&rb); dataLen += TD_ROW_LEN(rb.pBuf); #ifdef TD_DEBUG_PRINT_ROW tdSRowPrint(rb.pBuf, pTSchema, __func__); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 754e142437..6558af8aad 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -666,6 +666,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { ++iColVal; } + tdSRowEnd(&rb); taosMemoryFreeClear(varBuf); @@ -880,26 +881,20 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, int16_t colIdx) { -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { - output->valType = TD_VTYPE_NONE; - return terrno; - } - if (tdValTypeIsNorm(output->valType)) { + if (pRow->normal) { if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); } + return TSDB_CODE_SUCCESS; } -#else - if (IS_VAR_DATA_TYPE(colType)) { - output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + + if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { + output->valType = TD_VTYPE_NONE; + return terrno; } - output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif + return TSDB_CODE_SUCCESS; } @@ -925,6 +920,21 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp return TSDB_CODE_SUCCESS; } // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. + + switch (valType) { + case TD_VTYPE_NORM: + break; + case TD_VTYPE_NULL: + ++pBuilder->nNull; + break; + case TD_VTYPE_NONE: + ++pBuilder->nNone; + break; + default: + ASSERT(0); + break; + } + if (TD_IS_TP_ROW(pRow)) { tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset); } else { @@ -1084,6 +1094,14 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou return TSDB_CODE_SUCCESS; } +int32_t tdSRowEnd(SRowBuilder *pBuilder) { + STSRow *pRow = (STSRow *)pBuilder->pBuf; + if (pBuilder->nNone || pBuilder->nNull) { + pRow->normal = 1; + } + return TSDB_CODE_SUCCESS; +} + int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { @@ -1092,6 +1110,10 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { return terrno; } + if (pBuilder->nNone) pBuilder->nNone = 0; + if (pBuilder->nNull) pBuilder->nNull = 0; + + pBuilder->nNull = 0; TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); @@ -1159,14 +1181,6 @@ int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { return TSDB_CODE_SUCCESS; } -int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, - int32_t nBoundCols, int32_t flen) { - if (tdSRowSetExtendedInfo(pBuilder, allNullLen, boundNullLen, nCols, nBoundCols, flen) < 0) { - return terrno; - } - return tdSRowResetBuf(pBuilder, pBuf); -} - void tdSRowReset(SRowBuilder *pBuilder) { pBuilder->rowType = TD_ROW_TP; pBuilder->pBuf = NULL; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 850394b15d..fc6f2b0fa8 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -157,6 +157,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); } } + tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); rowData = POINTER_SHIFT(rowData, rowLen); dataLen += rowLen; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 1c08fafaa3..5e43e52afa 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -213,6 +213,7 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); } } + tdSRowEnd(&rb); if (ignoreRow) { continue; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index fbc5e1b4b0..d1cf82ff1f 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1206,6 +1206,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, } // *len = pBuilder->extendedRowSize; + tdSRowEnd(pBuilder); return TSDB_CODE_SUCCESS; } @@ -1973,6 +1974,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in } } } + tdSRowEnd(pBuilder); #ifdef TD_DEBUG_PRINT_ROW STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); tdSRowPrint(row, pSTSchema, __func__); @@ -2056,7 +2058,9 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu } } } - + if(rowEnd) { + tdSRowEnd(pBuilder); + } #ifdef TD_DEBUG_PRINT_ROW if (rowEnd) { STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); @@ -2436,6 +2440,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols } } + tdSRowEnd(pBuilder); pDataBlock->size += extendedRowSize; } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index ae123a3563..ee35b23cc0 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -659,6 +659,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB toffset += TYPE_BYTES[colType]; p += pSchema[j].bytes; } + tdSRowEnd(&builder); int32_t rowLen = TD_ROW_LEN((STSRow*)pDataBlock); pDataBlock = (char*)pDataBlock + rowLen; pBlock->dataLen += rowLen; From 0c4454c07764d99682d3d4d6992ba8357ef98427 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:23:27 +0800 Subject: [PATCH 014/165] fix: set statis to 0 when all col is normal --- include/common/trow.h | 2 +- source/common/src/trow.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 93004b2cbd..f2f68ddccd 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -38,7 +38,7 @@ typedef struct { uint16_t type : 2; uint16_t del : 1; uint16_t endian : 1; - uint16_t normal : 1; // all norm + uint16_t statis : 1; // 0 all normal, 1 has null or none uint16_t reserve : 11; uint16_t sver; }; diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6558af8aad..b54d653df9 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -881,7 +881,7 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, int16_t colIdx) { - if (pRow->normal) { + if (pRow->statis == 0) { if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { @@ -1097,7 +1097,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou int32_t tdSRowEnd(SRowBuilder *pBuilder) { STSRow *pRow = (STSRow *)pBuilder->pBuf; if (pBuilder->nNone || pBuilder->nNull) { - pRow->normal = 1; + pRow->statis = 1; } return TSDB_CODE_SUCCESS; } From 2b06d8328c711fdb537ac2bff21310b426311afb Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:27:50 +0800 Subject: [PATCH 015/165] enh: row end optimization --- include/common/trow.h | 7 ++++++- source/common/src/trow.c | 8 -------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index f2f68ddccd..35270341ff 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -290,7 +290,12 @@ int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen); int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, int32_t allNullLen, int32_t boundNullLen); int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); -int32_t tdSRowEnd(SRowBuilder *pBuilder); +static FORCE_INLINE void tdSRowEnd(SRowBuilder *pBuilder) { + STSRow *pRow = (STSRow *)pBuilder->pBuf; + if (pBuilder->nNone || pBuilder->nNull) { + pRow->statis = 1; + } +} int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf); void tdSRowReset(SRowBuilder *pBuilder); int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, diff --git a/source/common/src/trow.c b/source/common/src/trow.c index b54d653df9..ec51d65cdd 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -1094,14 +1094,6 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou return TSDB_CODE_SUCCESS; } -int32_t tdSRowEnd(SRowBuilder *pBuilder) { - STSRow *pRow = (STSRow *)pBuilder->pBuf; - if (pBuilder->nNone || pBuilder->nNull) { - pRow->statis = 1; - } - return TSDB_CODE_SUCCESS; -} - int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { From 22512c09222c82055038d92fd5e3581ba1fd1f78 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:33:53 +0800 Subject: [PATCH 016/165] enh: row optimization --- include/common/trow.h | 6 +++--- source/common/src/trow.c | 9 ++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 35270341ff..bd8510dde6 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -150,8 +150,8 @@ typedef struct { void *pBitmap; void *pOffset; int32_t extendedRowSize; - int16_t nNone; - int16_t nNull; + bool hasNone; + bool hasNull; } SRowBuilder; #define TD_ROW_HEAD_LEN (sizeof(STSRow)) @@ -292,7 +292,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); static FORCE_INLINE void tdSRowEnd(SRowBuilder *pBuilder) { STSRow *pRow = (STSRow *)pBuilder->pBuf; - if (pBuilder->nNone || pBuilder->nNull) { + if (pBuilder->hasNull || pBuilder->hasNone) { pRow->statis = 1; } } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index ec51d65cdd..d968221432 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -925,10 +925,10 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp case TD_VTYPE_NORM: break; case TD_VTYPE_NULL: - ++pBuilder->nNull; + if (!pBuilder->hasNull) pBuilder->hasNull = true; break; case TD_VTYPE_NONE: - ++pBuilder->nNone; + if (!pBuilder->hasNone) pBuilder->hasNone = true; break; default: ASSERT(0); @@ -1102,10 +1102,9 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { return terrno; } - if (pBuilder->nNone) pBuilder->nNone = 0; - if (pBuilder->nNull) pBuilder->nNull = 0; + if (pBuilder->hasNone) pBuilder->hasNone = false; + if (pBuilder->hasNull) pBuilder->hasNull = false; - pBuilder->nNull = 0; TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); From 258448bae9ca9b40775d47dd22fb76d7f1f0c320 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:42:08 +0800 Subject: [PATCH 017/165] fix: get normal val from tp row --- source/common/src/trow.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d968221432..a096db4b1d 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -882,6 +882,7 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, int16_t colIdx) { if (pRow->statis == 0) { + output->valType = TD_VTYPE_NORM; if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { @@ -895,6 +896,14 @@ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t return terrno; } + if (tdValTypeIsNorm(output->valType)) { + if (IS_VAR_DATA_TYPE(colType)) { + output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + } + return TSDB_CODE_SUCCESS; } From 843a516919eb5aa37add33e3cee09b9e8405e700 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:44:23 +0800 Subject: [PATCH 018/165] enh: get normal val from tp row --- source/common/src/trow.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index a096db4b1d..6e6b5acecf 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -896,7 +896,7 @@ int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t return terrno; } - if (tdValTypeIsNorm(output->valType)) { + if (output->valType == TD_VTYPE_NORM) { if (IS_VAR_DATA_TYPE(colType)) { output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); } else { From 846ee63c88f8eb52cc6f47dd9bd01d5c3ce90f81 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 3 Aug 2022 16:47:41 +0800 Subject: [PATCH 019/165] enh: normal val optimization --- source/common/src/trow.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6e6b5acecf..6f58a0ded1 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -912,7 +912,7 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp STSRow *pRow = pBuilder->pBuf; if (!val) { #ifdef TD_SUPPORT_BITMAP - if (tdValTypeIsNorm(valType)) { + if (valType == TD_VTYPE_NORM) { terrno = TSDB_CODE_INVALID_PTR; return terrno; } @@ -970,7 +970,7 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo STSRow *row = pBuilder->pBuf; // No need to store None/Null values. - if (tdValIsNorm(valType, val, colType)) { + if (valType == TD_VTYPE_NORM) { // ts key stored in STSRow.ts SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); @@ -1031,7 +1031,7 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo // 1. No need to set flen part for Null/None, just use bitmap. When upsert for the same primary TS key, the bitmap // should be updated simultaneously if Norm val overwrite Null/None cols. // 2. When consume STSRow in memory by taos client/tq, the output of Null/None cols should both be Null. - if (tdValIsNorm(valType, val, colType)) { + if (valType == TD_VTYPE_NORM) { // TODO: The layout of new data types imported since 3.0 like blob/medium blob is the same with binary/nchar. if (IS_VAR_DATA_TYPE(colType)) { // ts key stored in STSRow.ts From c8c9bbdeb1a6f6fa948a287be6ba3a4ec08a9222 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 3 Aug 2022 17:04:14 +0800 Subject: [PATCH 020/165] enh: last_row function scan in reverse order --- source/libs/planner/src/planOptimizer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index c02ef64707..ddf8b29153 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -242,7 +242,7 @@ static int32_t scanPathOptGetScanOrder(SScanLogicNode* pScan, EScanOrder* pScanO SFunctionNode* pFunc = (SFunctionNode*)pNode; if (FUNCTION_TYPE_FIRST == pFunc->funcType) { hasFirst = true; - } else if (FUNCTION_TYPE_LAST == pFunc->funcType) { + } else if (FUNCTION_TYPE_LAST == pFunc->funcType || FUNCTION_TYPE_LAST_ROW == pFunc->funcType) { hasLast = true; } else if (FUNCTION_TYPE_SELECT_VALUE != pFunc->funcType) { otherFunc = true; @@ -436,8 +436,8 @@ static int32_t pushDownCondOptDealScan(SOptimizeContext* pCxt, SScanLogicNode* p SNode* pPrimaryKeyCond = NULL; SNode* pOtherCond = NULL; - int32_t code = filterPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, &pScan->pTagCond, - &pOtherCond); + int32_t code = filterPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, + &pScan->pTagCond, &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pScan->pTagCond) { code = pushDownCondOptRebuildTbanme(&pScan->pTagCond); } From caa5378d8e2f2f88d64f00f53bf8ddbb711fdedd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 17:12:10 +0800 Subject: [PATCH 021/165] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b0f05e4705..93a842d24c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -296,7 +296,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; From 2a8da022def0cad9a04569ef44b72e60887a760b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 19:01:06 +0800 Subject: [PATCH 022/165] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 3 --- source/libs/function/src/builtins.c | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 93a842d24c..a6addc208d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -675,9 +675,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; -// pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose -// pInfo->cond.order = TSDB_ORDER_DESC; - pInfo->pdInfo.interval = extractIntervalInfo(pTableScanNode); pInfo->readHandle = *readHandle; pInfo->sample.sampleRatio = pTableScanNode->ratio; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 444cd2c5d0..1c519bae48 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2276,6 +2276,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = lastRowFunction, .sprocessFunc = firstLastScalarFunction, + .pPartialFunc = "_last_row_partial", + .pMergeFunc = "_last_row_merge", .finalizeFunc = firstLastFinalize }, { @@ -2288,6 +2290,27 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = cachedLastRowFunction, .finalizeFunc = firstLastFinalize }, + { + .name = "_last_row_partial", + .type = FUNCTION_TYPE_LAST_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .translateFunc = translateFirstLastPartial, + .dynDataRequiredFunc = lastDynDataReq, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastRowFunction, + .finalizeFunc = firstLastPartialFinalize, + }, + { + .name = "_last_row_merge", + .type = FUNCTION_TYPE_LAST_MERGE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .translateFunc = translateFirstLastMerge, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunctionMerge, + .finalizeFunc = firstLastFinalize, + }, { .name = "first", .type = FUNCTION_TYPE_FIRST, From c253ea73e1a6dd62870569465017ba6a77f5d8f1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 19:17:07 +0800 Subject: [PATCH 023/165] fix(query): fix the null not returns problem. --- source/libs/function/src/builtinsimpl.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 38c35e382a..a4e74e7a7d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2953,25 +2953,24 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) { SInputColumnInfoData* pColInfo = &pCtx->input; - int32_t start = pColInfo->startRowIndex; - - pOutput->bytes = pInput->bytes; - TSKEY* tsIn = &pInput->ts; - TSKEY* tsOut = &pOutput->ts; + int32_t start = pColInfo->startRowIndex; if (pOutput->hasResult) { if (isFirst) { - if (*tsIn > *tsOut) { + if (pInput->ts > pOutput->ts) { return; } } else { - if (*tsIn < *tsOut) { + if (pInput->ts < pOutput->ts) { return; } } } - *tsOut = *tsIn; + pOutput->isNull = pInput->isNull; + pOutput->ts = pInput->ts; + pOutput->bytes = pInput->bytes; + memcpy(pOutput->buf, pInput->buf, pOutput->bytes); saveTupleData(pCtx->pSrcBlock, start, pCtx, pOutput); @@ -2998,7 +2997,6 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer } SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); - return TSDB_CODE_SUCCESS; } @@ -3099,9 +3097,9 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { TSKEY startKey = getRowPTs(pInput->pPTS, 0); TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); +#if 0 int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; -#if 0 // the optimized version only function if all tuples in one block are monotonious increasing or descreasing. // this is NOT always works if project operator exists in downstream. if (blockDataOrder == TSDB_ORDER_ASC) { @@ -3141,6 +3139,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { } #endif + SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } From 0410959aacdae6b495fae1b547ecd47ee14ad9f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Aug 2022 19:19:17 +0800 Subject: [PATCH 024/165] test: disable some unit case. --- source/client/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 95e86e7629..f52edbe71f 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -46,6 +46,6 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) -#if(${BUILD_TEST}) +if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -#endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) \ No newline at end of file From 3566e8cf46913e7acc0e8be9266100d036e35096 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 15:02:05 +0800 Subject: [PATCH 025/165] enh(query): diff support timestamp column type. --- source/libs/function/src/builtins.c | 4 +--- source/libs/function/src/builtinsimpl.c | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1c519bae48..4ef2525b36 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1523,7 +1523,7 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType) { + if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType && TSDB_DATA_TYPE_TIMESTAMP != colType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -2204,8 +2204,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = elapsedFinalize, .invertFunc = NULL, .combineFunc = elapsedCombine, - //.pPartialFunc = "_elapsed_partial", - //.pMergeFunc = "_elapsed_merge" }, { .name = "_elapsed_partial", diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a4e74e7a7d..650aa3f12d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3170,6 +3170,8 @@ bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { switch (type) { case TSDB_DATA_TYPE_BOOL: + pDiffInfo->prev.i64 = *(bool*)pv? 1:0; + break; case TSDB_DATA_TYPE_TINYINT: pDiffInfo->prev.i64 = *(int8_t*)pv; break; @@ -3179,6 +3181,7 @@ static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { case TSDB_DATA_TYPE_SMALLINT: pDiffInfo->prev.i64 = *(int16_t*)pv; break; + case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: pDiffInfo->prev.i64 = *(int64_t*)pv; break; @@ -3232,6 +3235,7 @@ static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SCo pDiffInfo->prev.i64 = v; break; } + case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: { int64_t v = *(int64_t*)pv; int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null From 271528f4e21249c3e424a49d62634c1aed67d140 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 18:12:44 +0800 Subject: [PATCH 026/165] refactor: opt tsdb performance. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9b5523431c..7a867d85c9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2113,7 +2113,7 @@ TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pRea } TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); - TSDBKEY key = TSDBROW_KEY(pRow); + TSDBKEY key = {.ts = pRow->pTSRow->ts, .version = pRow->version}; if (outOfTimeWindow(key.ts, &pReader->window)) { pIter->hasVal = false; return NULL; From be1d1c5f8ecf6607151d8a539dcb9058e913b74f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 20:37:36 +0800 Subject: [PATCH 027/165] fix(query): clean temp files. --- source/libs/executor/src/groupoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 938134c167..528b39c17f 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -727,7 +727,7 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFree(pInfo->columnOffset); cleanupExprSupp(&pInfo->scalarSup); - + clearDiskbasedBuf(pInfo->pBuf); taosMemoryFreeClear(param); } From 27d223356a1df0a11a0c2d56801505e5c4923c3b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 4 Aug 2022 20:41:30 +0800 Subject: [PATCH 028/165] enh: merge multiple blocks in one rsp msg --- include/common/tmsg.h | 1 + include/libs/executor/dataSinkMgt.h | 1 + source/libs/qworker/inc/qwInt.h | 2 + source/libs/qworker/src/qwMsg.c | 9 ++- source/libs/qworker/src/qworker.c | 114 ++++++++++++++++++---------- 5 files changed, 83 insertions(+), 44 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bfb80ec8f8..9d001c9534 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1364,6 +1364,7 @@ typedef struct { int8_t compressed; int8_t streamBlockType; int32_t compLen; + int32_t numOfBlocks; int32_t numOfRows; int32_t numOfCols; int64_t skey; diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 47177dc11b..71105d88eb 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -67,6 +67,7 @@ typedef struct SInputData { } SInputData; typedef struct SOutputData { + int32_t numOfBlocks; int32_t numOfRows; int32_t numOfCols; int8_t compressed; diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 8f036714c9..729ac474e4 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -35,6 +35,7 @@ extern "C" { #define QW_DEFAULT_SHORT_RUN_TIMES 2 #define QW_DEFAULT_HEARTBEAT_MSEC 5000 #define QW_SCH_TIMEOUT_MSEC 180000 +#define QW_MIN_RES_ROWS 4096 enum { QW_PHASE_PRE_QUERY = 1, @@ -135,6 +136,7 @@ typedef struct SQWTaskCtx { int32_t msgType; int32_t fetchType; int32_t execId; + int32_t level; bool queryRsped; bool queryEnd; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index e8ffd98153..63a3c1bfea 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -12,14 +12,16 @@ int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { int32_t msgSize = sizeof(SRetrieveTableRsp) + length; - SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(msgSize); + SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcReallocCont(*rsp, msgSize); if (NULL == pRsp) { qError("rpcMallocCont %d failed", msgSize); QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - memset(pRsp, 0, sizeof(SRetrieveTableRsp)); - + if (NULL == *rsp) { + memset(pRsp, 0, sizeof(SRetrieveTableRsp)); + } + *rsp = pRsp; return TSDB_CODE_SUCCESS; @@ -35,6 +37,7 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) rsp->compLen = htonl(len); rsp->numOfRows = htonl(input->numOfRows); rsp->numOfCols = htonl(input->numOfCols); + rsp->numOfBlocks = htonl(input->numOfBlocks); } void qwFreeFetchRsp(void *msg) { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 36d85f1f12..d93c07ce1e 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -203,57 +203,88 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, SRetrieveTableRsp *rsp = NULL; bool queryEnd = false; int32_t code = 0; + SOutputData output = {0}; - dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); + *dataLen = 0; - if (len < 0) { - QW_TASK_ELOG("invalid length from dsGetDataLength, length:%d", len); - QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + while (true) { + dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); - if (len == 0) { - if (queryEnd) { - code = dsGetDataBlock(ctx->sinkHandle, pOutput); - if (code) { - QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); - } - - QW_TASK_DLOG_E("no data in sink and query end"); - - qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); - QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); - - *rspMsg = rsp; - *dataLen = 0; - return TSDB_CODE_SUCCESS; + if (len < 0) { + QW_TASK_ELOG("invalid length from dsGetDataLength, length:%d", len); + QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - pOutput->bufStatus = DS_BUF_EMPTY; + if (len == 0) { + if (queryEnd) { + code = dsGetDataBlock(ctx->sinkHandle, &output); + if (code) { + QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); + } - return TSDB_CODE_SUCCESS; + QW_TASK_DLOG("no more data in sink and query end, fetched blocks %d rows %d", pOutput->numOfBlocks, pOutput->numOfRows); + + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); + if (NULL == rsp) { + QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); + *pOutput = output; + } else { + pOutput->queryEnd = output.queryEnd; + pOutput->bufStatus = output.bufStatus; + pOutput->useconds = output.useconds; + } + + break; + } + + pOutput->bufStatus = DS_BUF_EMPTY; + + break; + } + + // Got data from sink + QW_TASK_DLOG("there are data in sink, dataLength:%d", len); + + *dataLen += len; + + QW_ERR_RET(qwMallocFetchRsp(*dataLen, &rsp)); + + output.pData = rsp->data + *dataLen - len; + code = dsGetDataBlock(ctx->sinkHandle, &output); + if (code) { + QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); + } + + pOutput->queryEnd = output.queryEnd; + pOutput->precision = output.precision; + pOutput->bufStatus = output.bufStatus; + pOutput->useconds = output.useconds; + pOutput->compressed = output.compressed; + pOutput->numOfCols = output.numOfCols; + pOutput->numOfRows += output.numOfRows; + pOutput->numOfBlocks++; + + if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { + QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %d", pOutput->numOfBlocks, pOutput->numOfRows); + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); + break; + } + + if (0 == ctx->level) { + QW_TASK_DLOG("task fetched blocks %d rows %d, level %d", pOutput->numOfBlocks, pOutput->numOfRows, ctx->level); + break; + } + + if (pOutput->numOfRows >= QW_MIN_RES_ROWS) { + QW_TASK_DLOG("task fetched blocks %d rows %d reaches the min rows", pOutput->numOfBlocks, pOutput->numOfRows); + break; + } } - // Got data from sink - QW_TASK_DLOG("there are data in sink, dataLength:%d", len); - - *dataLen = len; - - QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); *rspMsg = rsp; - pOutput->pData = rsp->data; - code = dsGetDataBlock(ctx->sinkHandle, pOutput); - if (code) { - QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); - } - - if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { - QW_TASK_DLOG_E("task all data fetched, done"); - qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); - } - return TSDB_CODE_SUCCESS; } @@ -551,6 +582,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) { // queryRsped = true; + ctx->level = plan->level; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); atomic_store_ptr(&ctx->sinkHandle, sinkHandle); From 569c82886f4d26488a896b43e785004b64202b56 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Aug 2022 20:42:23 +0800 Subject: [PATCH 029/165] fix(query): remove temp files. --- source/libs/executor/src/groupoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 528b39c17f..507719e0aa 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -727,7 +727,7 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFree(pInfo->columnOffset); cleanupExprSupp(&pInfo->scalarSup); - clearDiskbasedBuf(pInfo->pBuf); + destroyDiskbasedBuf(pInfo->pBuf); taosMemoryFreeClear(param); } From dc15a3dadf06d55531e037ff20e51d2df9e5a529 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 11:23:36 +0800 Subject: [PATCH 030/165] opt rpc send/recv --- source/libs/transport/inc/transComm.h | 16 ++----- source/libs/transport/src/transComm.c | 63 +++++---------------------- 2 files changed, 13 insertions(+), 66 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index a81d6db80f..853f24b0ce 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -127,7 +127,7 @@ typedef struct { int8_t retryCnt; int8_t retryLimit; - // bool setMaxRetry; + STransCtx appCtx; // STransMsg* pRsp; // for synchronous API tsem_t* pSem; // for synchronous API @@ -194,17 +194,7 @@ typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } Co #define transLabel(trans) ((STrans*)trans)->label -// int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); -// void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); -//// int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); -// -// int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); -// void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); -// bool transCompressMsg(char* msg, int32_t len, int32_t* flen); -// bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); - void transFreeMsg(void* msg); - // typedef struct SConnBuffer { char* buf; @@ -321,8 +311,8 @@ void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType); // request list typedef struct STransReq { - queue q; - void* data; + queue q; + uv_write_t wreq; } STransReq; void transReqQueueInit(queue* q); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 8cf525a506..ca405fa536 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -23,33 +23,6 @@ static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; static int32_t refMgt; static int32_t instMgt; -int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { - T_MD5_CTX context; - int ret = -1; - - tMD5Init(&context); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Update(&context, (uint8_t*)pMsg, msgLen); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Final(&context); - - if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; - - return ret; -} - -void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { - T_MD5_CTX context; - - tMD5Init(&context); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Update(&context, (uint8_t*)pMsg, msgLen); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Final(&context); - - memcpy(pAuth, context.digest, sizeof(context.digest)); -} - bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { return false; // SRpcHead* pHead = rpcHeadFromCont(pCont); @@ -176,7 +149,6 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { * info--->| */ SConnBuffer* p = connBuf; - uvBuf->base = p->buf + p->len; if (p->left == -1) { uvBuf->len = p->cap - p->len; @@ -267,14 +239,9 @@ int transAsyncSend(SAsyncPool* pool, queue* q) { uv_async_t* async = &(pool->asyncs[idx]); SAsyncItem* item = async->data; - int64_t st = taosGetTimestampUs(); taosThreadMutexLock(&item->mtx); QUEUE_PUSH(&item->qmsg, q); taosThreadMutexUnlock(&item->mtx); - int64_t el = taosGetTimestampUs() - st; - if (el > 50) { - // tInfo("lock and unlock cost:%d", (int)el); - } return uv_async_send(async); } @@ -350,30 +317,21 @@ void transReqQueueInit(queue* q) { QUEUE_INIT(q); } void* transReqQueuePush(queue* q) { - uv_write_t* req = taosMemoryCalloc(1, sizeof(uv_write_t)); - STransReq* wreq = taosMemoryCalloc(1, sizeof(STransReq)); - wreq->data = req; - req->data = wreq; - QUEUE_PUSH(q, &wreq->q); - return req; + STransReq* req = taosMemoryCalloc(1, sizeof(STransReq)); + req->wreq.data = req; + QUEUE_PUSH(q, &req->q); + return &req->wreq; } void* transReqQueueRemove(void* arg) { void* ret = NULL; - uv_write_t* req = arg; - STransReq* wreq = req && req->data ? req->data : NULL; + uv_write_t* wreq = arg; - assert(wreq->data == req); - if (wreq == NULL || wreq->data == NULL) { - taosMemoryFree(wreq->data); - taosMemoryFree(wreq); - return req; - } + STransReq* req = wreq ? wreq->data : NULL; + if (req == NULL) return NULL; + QUEUE_REMOVE(&req->q); - QUEUE_REMOVE(&wreq->q); - - ret = req && req->handle ? req->handle->data : NULL; - taosMemoryFree(wreq->data); - taosMemoryFree(wreq); + ret = wreq && wreq->handle ? wreq->handle->data : NULL; + taosMemoryFree(req); return ret; } @@ -382,7 +340,6 @@ void transReqQueueClear(queue* q) { queue* h = QUEUE_HEAD(q); QUEUE_REMOVE(h); STransReq* req = QUEUE_DATA(h, STransReq, q); - taosMemoryFree(req->data); taosMemoryFree(req); } } From 00d4efb4ea3dfb9cf94e404fa0c4d57186a3e677 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 11:28:49 +0800 Subject: [PATCH 031/165] opt rpc send --- source/libs/transport/src/transCli.c | 29 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 431e479123..140d2ef792 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1380,11 +1380,12 @@ int transReleaseCliHandle(void* handle) { } int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); - if (pTransInst == NULL) { - transFreeMsg(pReq->pCont); - return -1; - } + // STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + // if (pTransInst == NULL) { + // transFreeMsg(pReq->pCont); + // return -1; + // } + STrans* pTransInst = shandle; bool valid = false; SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); @@ -1404,7 +1405,6 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran if (ctx != NULL) { pCtx->appCtx = *ctx; } - assert(pTransInst->connType == TAOS_CONN_CLIENT); SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; @@ -1421,22 +1421,23 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return -1; } - transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return 0; } int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); - if (pTransInst == NULL) { - transFreeMsg(pReq->pCont); - return -1; - } + STrans* pTransInst = shandle; + // STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + // if (pTransInst == NULL) { + // transFreeMsg(pReq->pCont); + // return -1; + // } bool valid = false; SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); if (pThrd == NULL && valid == false) { transFreeMsg(pReq->pCont); - transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return -1; } @@ -1474,7 +1475,7 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs _RETURN: tsem_destroy(sem); taosMemoryFree(sem); - transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return ret; } /* From 46c2dfa452487ead247889fc29dd629b262770bf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 13:58:58 +0800 Subject: [PATCH 032/165] opt send --- source/libs/transport/src/trans.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 7633820292..5fa57fdbc2 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -80,7 +80,8 @@ void* rpcOpen(const SRpcInit* pInit) { int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); transAcquireExHandle(transGetInstMgt(), refId); pRpc->refId = refId; - return (void*)refId; + return pRpc; + // return (void*)refId; } void rpcClose(void* arg) { tInfo("start to close rpc"); From 0e4748771acd90aceeee5141b094bbce64e7d1ed Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 14:10:45 +0800 Subject: [PATCH 033/165] enh(query): support combine multi datablock to pass to upstream operators --- include/common/tdatablock.h | 2 +- source/client/src/clientImpl.c | 11 ++ source/common/src/tdatablock.c | 34 +++++- source/dnode/vnode/src/tsdb/tsdbRead.c | 7 +- source/libs/executor/inc/executorimpl.h | 9 +- source/libs/executor/src/dataDispatcher.c | 8 +- source/libs/executor/src/executorimpl.c | 142 +++++++++++++--------- source/libs/executor/src/scanoperator.c | 4 +- source/libs/stream/src/streamData.c | 5 +- 9 files changed, 142 insertions(+), 80 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 7839859e8b..3d605a144b 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -239,7 +239,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); -const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); +const char* blockDecode(SSDataBlock* pBlock, const char* pData); void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag); void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0c4dc1705c..8357e2d627 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1765,6 +1765,17 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t dataLen = *(int32_t*)p; p += sizeof(int32_t); + int32_t rows = *(int32_t*)p; + p += sizeof(int32_t); + + int32_t cols = *(int32_t*)p; + p += sizeof(int32_t); + + ASSERT(rows == numOfRows && cols == numOfCols); + + int32_t hasColumnSeg = *(int32_t*)p; + p += sizeof(int32_t); + uint64_t groupId = *(uint64_t*)p; p += sizeof(uint64_t); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4ce60efaea..bcbdf8d786 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -676,9 +676,9 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { * @return */ size_t blockDataGetSerialMetaSize(uint32_t numOfCols) { - // | total rows/total length | block group id | column schema | each column length | - return sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + - numOfCols * sizeof(int32_t); + // | total length | total rows | total columns | has column seg| block group id | column schema | each column length | + return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); } double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { @@ -2077,6 +2077,18 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ int32_t* actualLen = (int32_t*)data; data += sizeof(int32_t); + int32_t* rows = (int32_t*)data; + *rows = pBlock->info.rows; + data += sizeof(int32_t); + + int32_t* cols = (int32_t*)data; + *cols = numOfCols; + data += sizeof(int32_t); + + int32_t* hasColumnSegment = (int32_t*)data; + *hasColumnSegment = 1; + data += sizeof(int32_t); + uint64_t* groupId = (uint64_t*)data; data += sizeof(uint64_t); @@ -2130,12 +2142,26 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ *groupId = pBlock->info.groupId; } -const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { +const char* blockDecode(SSDataBlock* pBlock, const char* pData) { const char* pStart = pData; + // total length sizeof(int32_t) int32_t dataLen = *(int32_t*)pStart; pStart += sizeof(int32_t); + // total rows sizeof(int32_t) + int32_t numOfRows = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + // total columns sizeof(int32_t) + int32_t numOfCols = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + // has column info segment + int32_t hasColumnInfo = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + // group id sizeof(uint64_t) pBlock->info.groupId = *(uint64_t*)pStart; pStart += sizeof(uint64_t); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7a867d85c9..2a8e234ac0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2714,7 +2714,6 @@ void tsdbReaderClose(STsdbReader* pReader) { } SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap); taosMemoryFreeClear(pSupInfo->plist); @@ -2742,10 +2741,8 @@ void tsdbReaderClose(STsdbReader* pReader) { SIOCostSummary* pCost = &pReader->cost; tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 - " SMA-time:%.2f ms, " - "fileBlocks:%" PRId64 - ", fileBlocks-time:%.2f ms, build in-memory-block-time:%.2f ms, STableBlockScanInfo " - "size:%.2f Kb %s", + " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-time:%.2f ms, " + "build in-memory-block-time:%.2f ms, STableBlockScanInfo size:%.2f Kb %s", pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaData, pCost->smaLoadTime, pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 2e0ed802e5..62b2cc0fa6 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -266,7 +266,11 @@ typedef struct SExchangeInfo { SArray* pSourceDataInfo; tsem_t ready; void* pTransporter; - SSDataBlock* pResult; + // SArray, result block list, used to keep the multi-block that + // passed by downstream operator + SArray* pResultBlockList; + int32_t rspBlockIndex; // indicate the return block index in pResultBlockList + SSDataBlock* pDummyBlock; // dummy block, not keep data bool seqLoadData; // sequential load data or not, false by default int32_t current; SLoadRemoteDataInfo loadInfo; @@ -855,8 +859,7 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWin int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, - SArray* pColList); + int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart); STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index f0fb5852a0..3a37b5e760 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -69,10 +69,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { // clang-format off // data format: -// +----------------+--------------+-----------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) -// | |sizeof(int32) |sizeof(uint64_t) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | -// +----------------+--------------+-----------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// |SDataCacheEntry | total length | numOfRows | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) +// | |sizeof(int32) | sizeof(int32)| sizeof(uint64_t) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | +// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header // clang-format on diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 23dbd4d992..8dfda896cc 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1956,6 +1956,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->compLen = htonl(pRsp->compLen); pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); + pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); qDebug("%s fetch rsp received, index:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfRows); @@ -2043,12 +2044,10 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf } int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, - SArray* pColList) { + int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart) { if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); - // blockDataEnsureCapacity(pRes, numOfRows); - blockDecode(pRes, numOfOutput, numOfRows, pData); + *pNextStart = (char*) blockDecode(pRes, pData); } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; @@ -2072,7 +2071,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo blockDataAppendColInfo(pBlock, &idata); } - blockDecode(pBlock, numOfCols, numOfRows, pStart); + blockDecode(pBlock, pStart); blockDataEnsureCapacity(pRes, numOfRows); // data from mnode @@ -2084,8 +2083,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. blockDataUpdateTsWindow(pRes, 0); - int64_t el = taosGetTimestampUs() - startTs; - pLoadInfo->totalRows += numOfRows; pLoadInfo->totalSize += compLen; @@ -2093,7 +2090,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo *total += numOfRows; } - pLoadInfo->totalElapsed += el; return TSDB_CODE_SUCCESS; } @@ -2115,8 +2111,8 @@ static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { return NULL; } -static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, - SExecTaskInfo* pTaskInfo) { +static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, + SExecTaskInfo* pTaskInfo) { int32_t code = 0; int64_t startTs = taosGetTimestampUs(); size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); @@ -2142,7 +2138,6 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx SRetrieveTableRsp* pRsp = pDataInfo->pRsp; SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); - SSDataBlock* pRes = pExchangeInfo->pResult; SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pRsp->numOfRows == 0) { qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 @@ -2155,29 +2150,37 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx continue; } - SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; - code = - extractDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); - if (code != 0) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + int32_t index = 0; + char* pStart = pRetrieveRsp->data; + while(index++ < pRetrieveRsp->numOfBlocks) { + SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); + blockDataEnsureCapacity(pb, pRetrieveRsp->numOfRows); + code = + extractDataBlockFromFetchRsp(pb, pLoadInfo, pRetrieveRsp->numOfRows, pStart, + pRetrieveRsp->compLen, pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); + if (code != 0) { + taosMemoryFreeClear(pDataInfo->pRsp); + goto _error; + } + + taosArrayPush(pExchangeInfo->pResultBlockList, &pb); } + pLoadInfo->totalElapsed += (taosGetTimestampUs() - startTs); + if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d" - " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 - ", completed:%d try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources); + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d" + " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb," + " completed:%d try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfRows, + pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0, completed + 1, i + 1, totalSources); completed += 1; pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 - ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, - pLoadInfo->totalRows, pLoadInfo->totalSize); + ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, + pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0); } taosMemoryFreeClear(pDataInfo->pRsp); @@ -2191,11 +2194,12 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx } } - return pExchangeInfo->pResult; + return; } if (completed == totalSources) { - return setAllSourcesCompleted(pOperator, startTs); + setAllSourcesCompleted(pOperator, startTs); + return; } sched_yield(); @@ -2203,7 +2207,6 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx _error: pTaskInfo->code = code; - return NULL; } static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { @@ -2233,7 +2236,7 @@ static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { +static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2242,7 +2245,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { while (1) { if (pExchangeInfo->current >= totalSources) { - return setAllSourcesCompleted(pOperator, startTs); + setAllSourcesCompleted(pOperator, startTs); + return TSDB_CODE_SUCCESS; } doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); @@ -2255,7 +2259,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); pOperator->pTaskInfo->code = pDataInfo->code; - return NULL; + return pOperator->pTaskInfo->code; } SRetrieveTableRsp* pRsp = pDataInfo->pRsp; @@ -2272,16 +2276,16 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { continue; } - SSDataBlock* pRes = pExchangeInfo->pResult; - SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; - int32_t code = - extractDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + + char* pStart = pRetrieveRsp->data; + int32_t code = extractDataBlockFromFetchRsp(NULL, pLoadInfo, pRetrieveRsp->numOfRows, pStart, pRetrieveRsp->compLen, + pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources); @@ -2290,13 +2294,13 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize); } - pOperator->resultInfo.totalRows += pRes->info.rows; + pOperator->resultInfo.totalRows += pRetrieveRsp->numOfRows; taosMemoryFreeClear(pDataInfo->pRsp); - return pExchangeInfo->pResult; + return TSDB_CODE_SUCCESS; } } @@ -2320,6 +2324,11 @@ static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } +static void freeBlock(void* pParam) { + SSDataBlock* pBlock = *(SSDataBlock**)pParam; + blockDataDestroy(pBlock); +} + static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2329,9 +2338,9 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { return NULL; } - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pOperator->status == OP_EXEC_DONE) { qDebug("%s all %" PRIzu " source(s) are exhausted, total rows:%" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, @@ -2339,11 +2348,23 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { return NULL; } - if (pExchangeInfo->seqLoadData) { - return seqLoadRemoteData(pOperator); - } else { - return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList); + if (size == 0 || pExchangeInfo->rspBlockIndex >= size) { + pExchangeInfo->rspBlockIndex = 0; + taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock); + if (pExchangeInfo->seqLoadData) { + seqLoadRemoteData(pOperator); + } else { + concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + } + + if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { + return NULL; + } } + + // we have buffered retrieved datablock, return it directly + return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++); } static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { @@ -2360,26 +2381,24 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { return NULL; } - ASSERT(pBlock == pExchangeInfo->pResult); - SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; if (hasLimitOffsetInfo(pLimitInfo)) { - int32_t status = handleLimitOffset(pOperator, pLimitInfo, pExchangeInfo->pResult, false); + int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); if (status == PROJECT_RETRIEVE_CONTINUE) { continue; } else if (status == PROJECT_RETRIEVE_DONE) { - size_t rows = pExchangeInfo->pResult->info.rows; + size_t rows = pBlock->info.rows; pExchangeInfo->limitInfo.numOfOutputRows += rows; if (rows == 0) { doSetOperatorCompleted(pOperator); return NULL; } else { - return pExchangeInfo->pResult; + return pBlock; } } } else { - return pExchangeInfo->pResult; + return pBlock; } } } @@ -2442,16 +2461,18 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode } tsem_init(&pInfo->ready, 0, 0); + pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES); pInfo->seqLoadData = false; pInfo->pTransporter = pTransporter; - pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pResult->pDataBlock); + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, @@ -3585,12 +3606,15 @@ void doDestroyExchangeOperatorInfo(void* param) { taosArrayDestroy(pExInfo->pSources); taosArrayDestroy(pExInfo->pSourceDataInfo); - if (pExInfo->pResult != NULL) { - pExInfo->pResult = blockDataDestroy(pExInfo->pResult); + + if (pExInfo->pResultBlockList != NULL) { + taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock); + pExInfo->pResultBlockList = NULL; } - tsem_destroy(&pExInfo->ready); + blockDataDestroy(pExInfo->pDummyBlock); + tsem_destroy(&pExInfo->ready); taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8545abecfb..03c08a6bf5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2266,9 +2266,11 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } } + char* pStart = pRsp->data; extractDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pRsp->numOfRows, pRsp->data, pRsp->compLen, - pOperator->exprSupp.numOfExprs, startTs, NULL, pInfo->scanCols); + pOperator->exprSupp.numOfExprs, NULL, pInfo->scanCols, &pStart); + //startTs, // todo log the filter info doFilterResult(pInfo); taosMemoryFree(pRsp); diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 0bf6d4c921..c96854b198 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -27,10 +27,9 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock ASSERT(pReq->blockNum == taosArrayGetSize(pReq->dataLen)); for (int32_t i = 0; i < blockNum; i++) { - /*int32_t len = *(int32_t*)taosArrayGet(pReq->dataLen, i);*/ SRetrieveTableRsp* pRetrieve = taosArrayGetP(pReq->data, i); SSDataBlock* pDataBlock = taosArrayGet(pArray, i); - blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); + blockDecode(pDataBlock, pRetrieve->data); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); @@ -51,7 +50,7 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock taosArraySetSize(pArray, 1); SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); - blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); + blockDecode(pDataBlock, pRetrieve->data); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); From d0524389d9501723f48514255636358e437a90ce Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 5 Aug 2022 15:07:38 +0800 Subject: [PATCH 034/165] fix: plan problem with interval fill statement --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 1ebba8bb63..2e2c4fbff7 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -719,7 +719,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; - pWindow->node.groupAction = getGroupAction(pCxt, pSelect); + pWindow->node.groupAction = (NULL != pInterval->pFill ? GROUP_ACTION_KEEP : getGroupAction(pCxt, pSelect)); pWindow->node.requireDataOrder = pSelect->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_IN_BLOCK; pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; From 0ac1b85a0e72b399a08f56bbefd25c9634c8a4e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 15:09:43 +0800 Subject: [PATCH 035/165] refactor: add some logs. --- source/libs/executor/src/executorimpl.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8dfda896cc..d17fe7633f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1959,7 +1959,8 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfRows); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows); } else { pSourceDataInfo->code = code; qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); @@ -2171,16 +2172,17 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d" - " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb," + " index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb," " completed:%d try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfRows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0, completed + 1, i + 1, totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, + pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, + completed + 1, i + 1, totalSources); completed += 1; pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, - pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0); + pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize/1024.0); } taosMemoryFreeClear(pDataInfo->pRsp); From 52173e14e1b753f4a530f205f05024b01e89ff55 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 16:10:45 +0800 Subject: [PATCH 036/165] fix(query): add check for null ptr. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2a8e234ac0..0749d7b133 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2312,6 +2312,12 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe return; } else { // has next point in mem/imem TSDBROW* pNextRow = getValidRow(pIter, pDelList, pReader); + if (pNextRow == NULL) { + *pTSRow = current.pTSRow; + *freeTSRow = false; + return; + } + if (TSDBROW_KEY(¤t).ts != TSDBROW_KEY(pNextRow).ts) { *pTSRow = current.pTSRow; *freeTSRow = false; From 1063064a59916219b24605d0b89ddc02552ad747 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Aug 2022 17:40:40 +0800 Subject: [PATCH 037/165] enh(query): support filter by SMA. --- include/libs/scalar/filter.h | 2 +- source/libs/executor/src/executorimpl.c | 12 +-- source/libs/executor/src/scanoperator.c | 116 ++++++++++++++++-------- source/libs/scalar/src/filter.c | 12 +-- 4 files changed, 84 insertions(+), 58 deletions(-) diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index 44064f0a9f..1f1d9dea93 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -44,7 +44,7 @@ extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo); extern void filterFreeInfo(SFilterInfo *info); -extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); +extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t numOfCols, int32_t numOfRows); /* condition split interface */ int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d17fe7633f..1908e2a92d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1011,16 +1011,6 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { // } //} -// static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, -// SqlFunctionCtx *pCtx, int32_t numOfRows) { -// STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -// -// if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { -// return true; -// } -// -// return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows); -// } #if 0 static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { STimeWindow w = {0}; @@ -1215,7 +1205,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc } // current block has been discard due to filter applied -// if (!doFilterByBlockStatistics(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) { +// if (!doFilterByBlockSMA(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) { // pCost->skipBlocks += 1; // qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, // pBlockInfo->window.ekey, pBlockInfo->rows); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 03c08a6bf5..a2e0b68623 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,10 +13,11 @@ * along with this program. If not, see . */ +#include "os.h" #include "executorimpl.h" +#include "filter.h" #include "function.h" #include "functionMgt.h" -#include "os.h" #include "querynodes.h" #include "systable.h" #include "tname.h" @@ -227,6 +228,57 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* return TSDB_CODE_SUCCESS; } +static FORCE_INLINE bool doFilterByBlockSMA(const SNode* pFilterNode, SColumnDataAgg** pColsAgg, int32_t numOfCols, + int32_t numOfRows) { + if (pColsAgg == NULL || pFilterNode == NULL) { + return true; + } + + SFilterInfo* filter = NULL; + + // todo move to the initialization function + int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + bool keep = filterRangeExecute(filter, pColsAgg, numOfCols, numOfRows); + + filterFreeInfo(filter); + return keep; +} + +static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + bool allColumnsHaveAgg = true; + SColumnDataAgg** pColAgg = NULL; + + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + if (!allColumnsHaveAgg) { + return false; + } + + // if (allColumnsHaveAgg == true) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + + // todo create this buffer during creating operator + if (pBlock->pBlockAgg == NULL) { + pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); + if (pBlock->pBlockAgg == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + } + } + + for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->pColMatchInfo); ++i) { + SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); + if (!pColMatchInfo->output) { + continue; + } + pBlock->pBlockAgg[pColMatchInfo->targetSlotId] = pColAgg[i]; + } + + return true; +} + static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -236,6 +288,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pCost->totalBlocks += 1; pCost->totalRows += pBlock->info.rows; + bool loadSMA = false; *status = pInfo->dataBlockLoadFlag; if (pTableScanInfo->pFilterNode != NULL || @@ -259,41 +312,34 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; - - bool allColumnsHaveAgg = true; - SColumnDataAgg** pColAgg = NULL; - - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } - - if (allColumnsHaveAgg == true) { - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - - // todo create this buffer during creating operator - if (pBlock->pBlockAgg == NULL) { - pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); - } - - for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->pColMatchInfo); ++i) { - SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); - if (!pColMatchInfo->output) { - continue; - } - pBlock->pBlockAgg[pColMatchInfo->targetSlotId] = pColAgg[i]; - } - - return TSDB_CODE_SUCCESS; - } else { // failed to load the block sma data, data block statistics does not exist, load data block instead + loadSMA = true; // mark the operator of load sma; + bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); + if (!success) { // failed to load the block sma data, data block statistics does not exist, load data block instead *status = FUNC_DATA_REQUIRED_DATA_LOAD; } } ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD); - // todo filter data block according to the block sma data firstly + // try to filter data block according to sma info + if (pTableScanInfo->pFilterNode != NULL) { + if (!loadSMA) { + doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); + } + bool keep = doFilterByBlockSMA(pTableScanInfo->pFilterNode, pBlock->pBlockAgg, taosArrayGetSize(pBlock->pDataBlock), + pBlockInfo->rows); + if (!keep) { + qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + pCost->filterOutBlocks += 1; + (*status) = FUNC_DATA_REQUIRED_FILTEROUT; + + return TSDB_CODE_SUCCESS; + } + } + + // try to filter datablock according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), @@ -303,16 +349,6 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return TSDB_CODE_SUCCESS; } -#if 0 - if (!doFilterByBlockStatistics(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { - pCost->filterOutBlocks += 1; - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); - (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - return TSDB_CODE_SUCCESS; - } -#endif - pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; @@ -2722,7 +2758,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc // todo filter data block according to the block sma data firstly #if 0 - if (!doFilterByBlockStatistics(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { + if (!doFilterByBlockSMA(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { pCost->filterOutBlocks += 1; qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 04328fda9c..a6a47da2e5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3245,7 +3245,7 @@ _return: return code; } -bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows) { +bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { if (FILTER_EMPTY_RES(info)) { return false; } @@ -3261,7 +3261,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t int32_t index = -1; SFilterRangeCtx *ctx = info->colRange[k]; for(int32_t i = 0; i < numOfCols; ++i) { - if (pDataStatis[i].colId == ctx->colId) { + if (pDataStatis[i]->colId == ctx->colId) { index = i; break; } @@ -3277,13 +3277,13 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t break; } - if (pDataStatis[index].numOfNull <= 0) { + if (pDataStatis[index]->numOfNull <= 0) { if (ctx->isnull && !ctx->notnull && !ctx->isrange) { ret = false; break; } - } else if (pDataStatis[index].numOfNull > 0) { - if (pDataStatis[index].numOfNull == numOfRows) { + } else if (pDataStatis[index]->numOfNull > 0) { + if (pDataStatis[index]->numOfNull == numOfRows) { if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { ret = false; break; @@ -3297,7 +3297,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t } } - SColumnDataAgg* pDataBlockst = &pDataStatis[index]; + SColumnDataAgg* pDataBlockst = pDataStatis[index]; SFilterRangeNode *r = ctx->rs; float minv = 0; From a5420dfaad260a06907080b3540b0d1b56546c99 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 18:36:19 +0800 Subject: [PATCH 038/165] fix rpc perf --- source/libs/transport/src/transCli.c | 24 ++++++++++++++++-------- source/libs/transport/src/transSvr.c | 3 +-- source/libs/transport/test/svrBench.c | 17 ++++++++--------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 140d2ef792..11ba2a50d3 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -16,7 +16,8 @@ #include "transComm.h" typedef struct SConnList { - queue conn; + queue conn; + int32_t size; } SConnList; typedef struct SCliConn { @@ -518,15 +519,18 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { if (QUEUE_IS_EMPTY(&plist->conn)) { return NULL; } + + plist->size -= 1; queue* h = QUEUE_HEAD(&plist->conn); SCliConn* conn = QUEUE_DATA(h, SCliConn, q); conn->status = ConnNormal; QUEUE_REMOVE(&conn->q); QUEUE_INIT(&conn->q); - transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); - conn->task = NULL; - + if (conn->task != NULL) { + transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); + conn->task = NULL; + } return conn; } static void addConnToPool(void* pool, SCliConn* conn) { @@ -555,13 +559,17 @@ static void addConnToPool(void* pool, SCliConn* conn) { assert(conn->list != NULL); QUEUE_INIT(&conn->q); QUEUE_PUSH(&conn->list->conn, &conn->q); + conn->list->size += 1; + conn->task = NULL; assert(!QUEUE_IS_EMPTY(&conn->list->conn)); - STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg)); - arg->param1 = conn; - arg->param2 = thrd; - conn->task = transDQSched(thrd->timeoutQueue, doCloseIdleConn, arg, CONN_PERSIST_TIME(pTransInst->idleTime)); + if (conn->list->size >= 10) { + STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg)); + arg->param1 = conn; + arg->param2 = thrd; + conn->task = transDQSched(thrd->timeoutQueue, doCloseIdleConn, arg, CONN_PERSIST_TIME(pTransInst->idleTime)); + } } static int32_t allocConnRef(SCliConn* conn, bool update) { if (update) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 8b27d95e52..14b8b35478 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -75,7 +75,6 @@ typedef struct SWorkThrd { SAsyncPool* asyncPool; uv_prepare_t* prepare; queue msg; - TdThreadMutex msgMtx; queue conn; void* pTransInst; @@ -499,6 +498,7 @@ void uvWorkerAsyncCb(uv_async_t* handle) { tError("unexcept occurred, continue"); continue; } + // release handle to rpc init if (msg->type == Quit) { (*transAsyncHandle[msg->type])(msg, pThrd); @@ -743,7 +743,6 @@ static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { pThrd->pipe->data = pThrd; QUEUE_INIT(&pThrd->msg); - taosThreadMutexInit(&pThrd->msgMtx, NULL); pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); uv_prepare_init(pThrd->loop, pThrd->prepare); diff --git a/source/libs/transport/test/svrBench.c b/source/libs/transport/test/svrBench.c index 224f527385..6eb80c8504 100644 --- a/source/libs/transport/test/svrBench.c +++ b/source/libs/transport/test/svrBench.c @@ -75,15 +75,14 @@ void processShellMsg() { void *handle = pRpcMsg->info.handle; taosFreeQitem(pRpcMsg); - - { - SRpcMsg nRpcMsg = {0}; - nRpcMsg.pCont = rpcMallocCont(msgSize); - nRpcMsg.contLen = msgSize; - nRpcMsg.info.handle = handle; - nRpcMsg.code = TSDB_CODE_CTG_NOT_READY; - rpcSendResponse(&nRpcMsg); - } + //{ + // SRpcMsg nRpcMsg = {0}; + // nRpcMsg.pCont = rpcMallocCont(msgSize); + // nRpcMsg.contLen = msgSize; + // nRpcMsg.info.handle = handle; + // nRpcMsg.code = TSDB_CODE_CTG_NOT_READY; + // rpcSendResponse(&nRpcMsg); + //} } taosUpdateItemSize(qinfo.queue, numOfMsgs); From f97b972191809130052c0ea5a20948889a3b7033 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 20:55:01 +0800 Subject: [PATCH 039/165] opt rpc perf --- source/libs/transport/src/trans.c | 3 +-- source/libs/transport/src/transCli.c | 32 +++++++++++++-------------- source/libs/transport/src/transComm.c | 3 ++- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 5fa57fdbc2..7633820292 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -80,8 +80,7 @@ void* rpcOpen(const SRpcInit* pInit) { int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); transAcquireExHandle(transGetInstMgt(), refId); pRpc->refId = refId; - return pRpc; - // return (void*)refId; + return (void*)refId; } void rpcClose(void* arg) { tInfo("start to close rpc"); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 11ba2a50d3..61888db74e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -564,7 +564,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { conn->task = NULL; assert(!QUEUE_IS_EMPTY(&conn->list->conn)); - if (conn->list->size >= 10) { + if (conn->list->size >= 50) { STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg)); arg->param1 = conn; arg->param2 = thrd; @@ -1372,7 +1372,7 @@ int transReleaseCliHandle(void* handle) { } STransMsg tmsg = {.info.handle = handle}; - TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64()); + // TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64()); SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cmsg->msg = tmsg; @@ -1388,12 +1388,11 @@ int transReleaseCliHandle(void* handle) { } int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - // STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); - // if (pTransInst == NULL) { - // transFreeMsg(pReq->pCont); - // return -1; - // } - STrans* pTransInst = shandle; + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + transFreeMsg(pReq->pCont); + return -1; + } bool valid = false; SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); @@ -1429,23 +1428,22 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return -1; } - // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return 0; } int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = shandle; - // STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); - // if (pTransInst == NULL) { - // transFreeMsg(pReq->pCont); - // return -1; - // } + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + transFreeMsg(pReq->pCont); + return -1; + } bool valid = false; SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); if (pThrd == NULL && valid == false) { transFreeMsg(pReq->pCont); - // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return -1; } @@ -1483,7 +1481,7 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs _RETURN: tsem_destroy(sem); taosMemoryFree(sem); - // transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return ret; } /* diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index ca405fa536..f0b1885d66 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -156,7 +156,8 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { if (p->left < p->cap - p->len) { uvBuf->len = p->left; } else { - p->buf = taosMemoryRealloc(p->buf, p->left + p->len); + p->cap = p->left + p->len; + p->buf = taosMemoryRealloc(p->buf, p->cap); uvBuf->base = p->buf + p->len; uvBuf->len = p->left; } From bab1b9c62196792a497a0b0bc6bcef394c77387e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 21:18:12 +0800 Subject: [PATCH 040/165] fix rpc perf --- source/util/src/tref.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 9cd849b9be..c984ef3f34 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -44,11 +44,11 @@ typedef struct { void (*fp)(void *); } SRefSet; -static SRefSet tsRefSetList[TSDB_REF_OBJECTS]; +static SRefSet tsRefSetList[TSDB_REF_OBJECTS]; static TdThreadOnce tsRefModuleInit = PTHREAD_ONCE_INIT; static TdThreadMutex tsRefMutex; -static int32_t tsRefSetNum = 0; -static int32_t tsNextId = 0; +static int32_t tsRefSetNum = 0; +static int32_t tsNextId = 0; static void taosInitRefModule(void); static void taosLockList(int64_t *lockedBy); From eeb71acdf661ffcd36ae53932082d90d82084b2c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 5 Aug 2022 21:28:34 +0800 Subject: [PATCH 041/165] fix rpc perf --- include/util/tdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 6d893765fc..a7747b9b45 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -396,7 +396,7 @@ typedef enum ELogicConditionType { #ifdef WINDOWS #define TSDB_MAX_RPC_THREADS 4 // windows pipe only support 4 connections. #else -#define TSDB_MAX_RPC_THREADS 5 +#define TSDB_MAX_RPC_THREADS 10 #endif #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type From 7f8195b3a588e467e1e3efeb1ba9e7c8adbfb252 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 6 Aug 2022 11:27:13 +0800 Subject: [PATCH 042/165] fix uninited value --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 61888db74e..3206a2ff39 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -510,7 +510,7 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { SHashObj* pPool = pool; SConnList* plist = taosHashGet(pPool, key, strlen(key)); if (plist == NULL) { - SConnList list; + SConnList list = {0}; taosHashPut(pPool, key, strlen(key), (void*)&list, sizeof(list)); plist = taosHashGet(pPool, key, strlen(key)); QUEUE_INIT(&plist->conn); From c7b511ed698b8addf86dc822d29ce961ba8e020e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 6 Aug 2022 16:13:18 +0800 Subject: [PATCH 043/165] fix(query): set src block for function. --- source/libs/executor/src/executorimpl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1908e2a92d..14914b51c0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -434,6 +434,7 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC pCtx[i].order = order; pCtx[i].input.numOfRows = pBlock->info.rows; setBlockStatisInfo(&pCtx[i], &pOperator->exprSupp.pExprInfo[i], pBlock); + pCtx[i].pSrcBlock = pBlock; } } From ebe8192d7ecd526546c0eab9a5c11087de5bda88 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 6 Aug 2022 16:15:15 +0800 Subject: [PATCH 044/165] refactor(query): do some internal refactor. --- source/libs/executor/src/executorimpl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 14914b51c0..b0929a9acd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -88,7 +88,7 @@ static int32_t getExprFunctionId(SExprInfo* pExprInfo) { static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); -static void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pSDataBlock); +static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pSDataBlock); static void releaseQueryBuf(size_t numOfTables); @@ -433,7 +433,7 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { pCtx[i].order = order; pCtx[i].input.numOfRows = pBlock->info.rows; - setBlockStatisInfo(&pCtx[i], &pOperator->exprSupp.pExprInfo[i], pBlock); + setBlockSMAInfo(&pCtx[i], &pOperator->exprSupp.pExprInfo[i], pBlock); pCtx[i].pSrcBlock = pBlock; } } @@ -799,7 +799,7 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF return TSDB_CODE_SUCCESS; } -void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pBlock) { +void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pBlock) { int32_t numOfRows = pBlock->info.rows; SInputColumnInfoData* pInput = &pCtx->input; From 7fc09d84752979921bad2445c447471272f20ab8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 6 Aug 2022 20:51:21 +0800 Subject: [PATCH 045/165] start timer for particular msg --- source/libs/transport/src/transCli.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 48d1829aa2..bd6fcac806 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -340,8 +340,8 @@ void cliHandleResp(SCliConn* conn) { tDebug("%s conn %p stop timer", CONN_GET_INST_LABEL(conn), conn); uv_timer_stop(conn->timer); } - conn->timer->data = NULL; taosArrayPush(pThrd->timerList, &conn->timer); + conn->timer->data = NULL; conn->timer = NULL; } @@ -483,6 +483,7 @@ void cliReadTimeoutCb(uv_timer_t* handle) { // set up timeout cb SCliConn* conn = handle->data; tTrace("%s conn %p timeout, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); + uv_read_stop(conn->stream); cliHandleExceptImpl(conn, TSDB_CODE_RPC_TIMEOUT); } @@ -542,6 +543,13 @@ static void addConnToPool(void* pool, SCliConn* conn) { allocConnRef(conn, true); + if (conn->timer != NULL) { + uv_timer_stop(conn->timer); + taosArrayPush(thrd->timerList, &conn->timer); + conn->timer->data = NULL; + conn->timer = NULL; + } + STrans* pTransInst = thrd->pTransInst; cliReleaseUnfinishedMsg(conn); transQueueClear(&conn->cliMsgs); From 6930b393d60eca1edf887ae454b179cf2d20146c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 6 Aug 2022 20:53:42 +0800 Subject: [PATCH 046/165] start timer for particular msg --- include/libs/transport/trpc.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 467f0a9ff0..655c903c0b 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -47,8 +47,6 @@ typedef struct SRpcHandleInfo { int8_t persistHandle; // persist handle or not int8_t hasEpSet; - STraceId traceId; - // app info void *ahandle; // app handle set by client void *wrapper; // wrapper handle @@ -58,7 +56,8 @@ typedef struct SRpcHandleInfo { void *rsp; int32_t rspLen; - // conn info + STraceId traceId; + SRpcConnInfo conn; } SRpcHandleInfo; From 5ae9fdff7a59bee3c685c147d142235650335770 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 8 Aug 2022 10:00:10 +0800 Subject: [PATCH 047/165] enh: increase explain output, enhance debugging function --- source/libs/command/inc/commandInt.h | 5 +++ source/libs/command/src/explain.c | 31 +++++++++++++++-- source/libs/function/src/builtins.c | 7 ++-- source/libs/parser/src/parInsertData.c | 6 ++-- source/libs/parser/src/parTranslater.c | 2 -- source/libs/planner/src/planLogicCreater.c | 40 ++++++++++++++-------- source/libs/planner/src/planPhysiCreater.c | 2 +- 7 files changed, 67 insertions(+), 26 deletions(-) diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index c53d464eab..e2288ed3ef 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -43,6 +43,7 @@ extern "C" { #define EXPLAIN_GROUP_SORT_FORMAT "Group Sort" #define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s" #define EXPLAIN_MERGE_INTERVAL_FORMAT "Merge Interval on Column %s" +#define EXPLAIN_MERGE_ALIGNED_INTERVAL_FORMAT "Merge Aligned Interval on Column %s" #define EXPLAIN_FILL_FORMAT "Fill" #define EXPLAIN_SESSION_FORMAT "Session" #define EXPLAIN_STATE_WINDOW_FORMAT "StateWindow on Column %s" @@ -59,6 +60,8 @@ extern "C" { #define EXPLAIN_RATIO_TIME_FORMAT "Ratio: %f" #define EXPLAIN_MERGE_FORMAT "SortMerge" #define EXPLAIN_MERGE_KEYS_FORMAT "Merge Key: " +#define EXPLAIN_PARTITION_KETS_FORMAT "Partition Key: " +#define EXPLAIN_INTERP_FORMAT "Interp" #define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms" #define EXPLAIN_EXEC_TIME_FORMAT "Execution Time: %.3f ms" @@ -69,6 +72,7 @@ extern "C" { #define EXPLAIN_LEFT_PARENTHESIS_FORMAT " (" #define EXPLAIN_RIGHT_PARENTHESIS_FORMAT ")" #define EXPLAIN_BLANK_FORMAT " " +#define EXPLAIN_COMMA_FORMAT ", " #define EXPLAIN_COST_FORMAT "cost=%.2f..%.2f" #define EXPLAIN_ROWS_FORMAT "rows=%" PRIu64 #define EXPLAIN_COLUMNS_FORMAT "columns=%d" @@ -86,6 +90,7 @@ extern "C" { #define EXPLAIN_OUTPUT_ORDER_TYPE_FORMAT "output_order=%s" #define EXPLAIN_OFFSET_FORMAT "offset=%d" #define EXPLAIN_SOFFSET_FORMAT "soffset=%d" +#define EXPLAIN_PARTITIONS_FORMAT "partitions=%d" #define COMMAND_RESET_LOG "resetLog" #define COMMAND_SCHEDULE_POLICY "schedulePolicy" diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 2836bd7acd..5935b19173 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -535,6 +535,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (NULL != pTblScanNode->pGroupTags) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PARTITION_KETS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_PARTITIONS_FORMAT, pTblScanNode->pGroupTags->length); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, @@ -612,13 +619,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pPrjNode->mergeDataBlock? "True":"False"); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pPrjNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -915,7 +926,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: { SMergeAlignedIntervalPhysiNode *pIntNode = (SMergeAlignedIntervalPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); + EXPLAIN_ROW_NEW(level, EXPLAIN_MERGE_ALIGNED_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); @@ -1129,6 +1140,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PARTITION_KETS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_PARTITIONS_FORMAT, pPartNode->pPartitionKeys->length); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pPartNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pPartNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, @@ -1201,11 +1217,20 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGE_KEYS_FORMAT); + if (pMergeNode->groupSort) { + EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, "_group_id asc"); + if (LIST_LENGTH(pMergeNode->pMergeKeys) > 0) { + EXPLAIN_ROW_APPEND(EXPLAIN_COMMA_FORMAT); + } + } for (int32_t i = 0; i < LIST_LENGTH(pMergeNode->pMergeKeys); ++i) { SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pMergeNode->pMergeKeys, i); EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, nodesGetNameFromColumnNode(ptn->pExpr)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, EXPLAIN_ORDER_STRING(ptn->order)); + if (i != LIST_LENGTH(pMergeNode->pMergeKeys) - 1) { + EXPLAIN_ROW_APPEND(EXPLAIN_COMMA_FORMAT); + } } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -1417,7 +1442,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: { SInterpFuncPhysiNode *pInterpNode = (SInterpFuncPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT); + EXPLAIN_ROW_NEW(level, EXPLAIN_INTERP_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bdd9232ce3..13ebcd4518 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1575,7 +1575,8 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType && TSDB_DATA_TYPE_TIMESTAMP != colType) { + if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType && + TSDB_DATA_TYPE_TIMESTAMP != colType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -2150,7 +2151,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "_apercentile_partial", - .type = FUNCTION_TYPE_APERCENTILE_PARTIAL | FUNC_MGT_TIMELINE_FUNC, + .type = FUNCTION_TYPE_APERCENTILE_PARTIAL, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateApercentilePartial, .getEnvFunc = getApercentileFuncEnv, @@ -2163,7 +2164,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_apercentile_merge", .type = FUNCTION_TYPE_APERCENTILE_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateApercentileMerge, .getEnvFunc = getApercentileFuncEnv, .initFunc = apercentileFunctionSetup, diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index ee35b23cc0..cd30672baf 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -486,11 +486,11 @@ static int32_t tdBlockRowMerge(STableMeta* pTableMeta, SBlockKeyTuple* pEndKeyTp void* pDestRow = NULL; ++((*pBlkRowMerger)->index); if ((*pBlkRowMerger)->index < taosArrayGetSize((*pBlkRowMerger)->rowArray)) { - void* pAlloc = *(void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index); - if (tRealloc((uint8_t**)&pAlloc, rowSize) != 0) { + void** pAlloc = (void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index); + if (tRealloc((uint8_t**)pAlloc, rowSize) != 0) { return TSDB_CODE_FAILED; } - pDestRow = pAlloc; + pDestRow = *pAlloc; } else { if (tRealloc((uint8_t**)&pDestRow, rowSize) != 0) { return TSDB_CODE_FAILED; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d96cdca8a9..2f6725acaf 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2543,7 +2543,6 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } - pSelect->isTimeLineResult = true; pCxt->currClause = SQL_CLAUSE_WINDOW; int32_t code = translateExpr(pCxt, &pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { @@ -2614,7 +2613,6 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec if (NULL == pSelect->pPartitionByList) { return TSDB_CODE_SUCCESS; } - pSelect->isTimeLineResult = false; pCxt->currClause = SQL_CLAUSE_PARTITION_BY; return translateExprList(pCxt, pSelect->pPartitionByList); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 2e2c4fbff7..98b97765de 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -342,7 +342,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pJoin->inputTsOrder = ORDER_ASC; pJoin->node.groupAction = GROUP_ACTION_CLEAR; pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; - pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; + pJoin->node.resultDataOrder = DATA_ORDER_LEVEL_GLOBAL; int32_t code = TSDB_CODE_SUCCESS; @@ -472,6 +472,11 @@ static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; } +static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { + return needTimeline ? (NULL != pSelect->pPartitionByList ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) + : DATA_ORDER_LEVEL_NONE; +} + static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; @@ -486,8 +491,8 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->hasTimeLineFunc = pSelect->hasTimeLineFunc; pAgg->onlyHasKeepOrderFunc = pSelect->onlyHasKeepOrderFunc; pAgg->node.groupAction = getGroupAction(pCxt, pSelect); - pAgg->node.requireDataOrder = pAgg->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; - pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; + pAgg->node.requireDataOrder = getRequireDataOrder(pAgg->hasTimeLineFunc, pSelect); + pAgg->node.resultDataOrder = pAgg->onlyHasKeepOrderFunc ? pAgg->node.requireDataOrder : DATA_ORDER_LEVEL_NONE; int32_t code = TSDB_CODE_SUCCESS; @@ -557,8 +562,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt pIdfRowsFunc->isUniqueFunc = pSelect->hasUniqueFunc; pIdfRowsFunc->isTimeLineFunc = pSelect->hasTimeLineFunc; pIdfRowsFunc->node.groupAction = getGroupAction(pCxt, pSelect); - pIdfRowsFunc->node.requireDataOrder = - pIdfRowsFunc->isTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; + pIdfRowsFunc->node.requireDataOrder = getRequireDataOrder(pIdfRowsFunc->isTimeLineFunc, pSelect); pIdfRowsFunc->node.resultDataOrder = pIdfRowsFunc->node.requireDataOrder; // indefinite rows functions and _select_values functions @@ -592,7 +596,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p } pInterpFunc->node.groupAction = getGroupAction(pCxt, pSelect); - pInterpFunc->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pInterpFunc->node.requireDataOrder = getRequireDataOrder(true, pSelect); pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder; int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsInterpFunc, &pInterpFunc->pFuncs); @@ -668,8 +672,10 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo pWindow->winType = WINDOW_TYPE_STATE; pWindow->node.groupAction = getGroupAction(pCxt, pSelect); - pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; - pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : getRequireDataOrder(true, pSelect); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : pWindow->node.requireDataOrder; pWindow->pStateExpr = nodesCloneNode(pState->pExpr); pWindow->pTspk = nodesCloneNode(pState->pCol); if (NULL == pWindow->pStateExpr || NULL == pWindow->pTspk) { @@ -691,8 +697,10 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i; pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? SESSION_ALGO_STREAM_SINGLE : SESSION_ALGO_MERGE; pWindow->node.groupAction = getGroupAction(pCxt, pSelect); - pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; - pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : getRequireDataOrder(true, pSelect); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : pWindow->node.requireDataOrder; pWindow->pTspk = nodesCloneNode((SNode*)pSession->pCol); if (NULL == pWindow->pTspk) { @@ -720,8 +728,12 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; pWindow->node.groupAction = (NULL != pInterval->pFill ? GROUP_ACTION_KEEP : getGroupAction(pCxt, pSelect)); - pWindow->node.requireDataOrder = pSelect->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_IN_BLOCK; - pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery + ? DATA_ORDER_LEVEL_IN_BLOCK + : (pSelect->hasTimeLineFunc ? getRequireDataOrder(true, pSelect) : DATA_ORDER_LEVEL_IN_BLOCK); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : getRequireDataOrder(true, pSelect); pWindow->pTspk = nodesCloneNode(pInterval->pCol); if (NULL == pWindow->pTspk) { @@ -768,8 +780,8 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } pFill->node.groupAction = getGroupAction(pCxt, pSelect); - pFill->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; - pFill->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pFill->node.requireDataOrder = getRequireDataOrder(true, pSelect); + pFill->node.resultDataOrder = pFill->node.requireDataOrder; pFill->inputTsOrder = ORDER_ASC; int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_WINDOW, NULL, COLLECT_COL_TYPE_ALL, &pFill->node.pTargets); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 4fe229269a..ff2e2f1494 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -985,7 +985,7 @@ static bool projectCanMergeDataBlock(SProjectLogicNode* pProject) { return true; } if (1 != LIST_LENGTH(pProject->node.pChildren)) { - return false; + return true; } SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProject->node.pChildren, 0); return DATA_ORDER_LEVEL_GLOBAL == pChild->resultDataOrder ? true : false; From 447b2cb9f1b8cd84e6e1c75890488c0ee9367e3d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Aug 2022 12:56:03 +0800 Subject: [PATCH 048/165] fix(query): update the load data log. --- source/libs/executor/inc/executorimpl.h | 6 ++-- source/libs/executor/src/executorimpl.c | 37 ++++++++++++------------- source/libs/executor/src/scanoperator.c | 5 ++-- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 62b2cc0fa6..1b8be3b53c 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -858,8 +858,10 @@ void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLi void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart); +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, char** pNextStart); +void updateLoadRemoteInfo(SLoadRemoteDataInfo *pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator); + STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b0929a9acd..752b93bcea 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2035,8 +2035,16 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf return TSDB_CODE_SUCCESS; } -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, uint64_t* total, SArray* pColList, char** pNextStart) { +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator) { + pInfo->totalRows += numOfRows; + pInfo->totalSize += dataLen; + pInfo->totalElapsed += (taosGetTimestampUs() - startTs); + pOperator->resultInfo.totalRows += numOfRows; +} + +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, + char** pNextStart) { if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); *pNextStart = (char*) blockDecode(pRes, pData); @@ -2064,24 +2072,16 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo } blockDecode(pBlock, pStart); - blockDataEnsureCapacity(pRes, numOfRows); + blockDataEnsureCapacity(pRes, pBlock->info.rows); // data from mnode - pRes->info.rows = numOfRows; + pRes->info.rows = pBlock->info.rows; relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); blockDataDestroy(pBlock); } // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. blockDataUpdateTsWindow(pRes, 0); - - pLoadInfo->totalRows += numOfRows; - pLoadInfo->totalSize += compLen; - - if (total != NULL) { - *total += numOfRows; - } - return TSDB_CODE_SUCCESS; } @@ -2148,9 +2148,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn while(index++ < pRetrieveRsp->numOfBlocks) { SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); blockDataEnsureCapacity(pb, pRetrieveRsp->numOfRows); - code = - extractDataBlockFromFetchRsp(pb, pLoadInfo, pRetrieveRsp->numOfRows, pStart, - pRetrieveRsp->compLen, pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); + code = extractDataBlockFromFetchRsp(pb, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); goto _error; @@ -2159,7 +2157,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn taosArrayPush(pExchangeInfo->pResultBlockList, &pb); } - pLoadInfo->totalElapsed += (taosGetTimestampUs() - startTs); + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d" @@ -2272,8 +2270,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; char* pStart = pRetrieveRsp->data; - int32_t code = extractDataBlockFromFetchRsp(NULL, pLoadInfo, pRetrieveRsp->numOfRows, pStart, pRetrieveRsp->compLen, - pRetrieveRsp->numOfCols, &pDataInfo->totalRows, NULL, &pStart); + int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 @@ -2291,7 +2288,9 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { pLoadInfo->totalRows, pLoadInfo->totalSize); } - pOperator->resultInfo.totalRows += pRetrieveRsp->numOfRows; + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); + pDataInfo->totalRows += pRetrieveRsp->numOfRows; + taosMemoryFreeClear(pDataInfo->pRsp); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a2e0b68623..0bd0d68cae 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2303,10 +2303,9 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } char* pStart = pRsp->data; - extractDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pRsp->numOfRows, pRsp->data, pRsp->compLen, - pOperator->exprSupp.numOfExprs, NULL, pInfo->scanCols, &pStart); + extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pOperator->exprSupp.numOfExprs, pInfo->scanCols, &pStart); + updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); - //startTs, // todo log the filter info doFilterResult(pInfo); taosMemoryFree(pRsp); From 200a669542175755a1f6d4bc669f41f2d4760047 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Aug 2022 16:34:11 +0800 Subject: [PATCH 049/165] refactor: do some internal refactor. --- include/common/tmsg.h | 6 ++--- source/client/src/clientImpl.c | 6 ++++- source/common/src/tdatablock.c | 34 +++++++++++++++++-------- source/libs/executor/src/executorimpl.c | 6 ++++- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9d001c9534..5ca6a337a8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -296,13 +296,13 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_IDX_ON ((int8_t)0x2) #define COL_SET_NULL ((int8_t)0x10) #define COL_SET_VAL ((int8_t)0x20) -typedef struct SSchema { +struct SSchema { int8_t type; int8_t flags; col_id_t colId; int32_t bytes; char name[TSDB_COL_NAME_LEN]; -} SSchema; +}; #define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0) #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) @@ -648,7 +648,7 @@ typedef struct { }; bool output; // TODO remove it later - int16_t type; + int8_t type; int32_t bytes; uint8_t precision; uint8_t scale; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8357e2d627..480fd99c81 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1762,6 +1762,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 char* p = (char*)pResultInfo->pData; + // version: + int32_t blockVersion = *(int32_t*)p; + p += sizeof(int32_t); + int32_t dataLen = *(int32_t*)p; p += sizeof(int32_t); @@ -1782,7 +1786,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 // check fields for (int32_t i = 0; i < numOfCols; ++i) { int16_t type = *(int16_t*)p; - p += sizeof(int16_t); + p += sizeof(int8_t); int32_t bytes = *(int32_t*)p; p += sizeof(int32_t); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index bcbdf8d786..45bbe96923 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -676,9 +676,9 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { * @return */ size_t blockDataGetSerialMetaSize(uint32_t numOfCols) { - // | total length | total rows | total columns | has column seg| block group id | column schema | each column length | - return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + - numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | + return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + + numOfCols * (sizeof(int8_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); } double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { @@ -1582,7 +1582,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { for (int32_t i = 0; i < sz; i++) { SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); tlen += taosEncodeFixedI16(buf, pColData->info.colId); - tlen += taosEncodeFixedI16(buf, pColData->info.type); + tlen += taosEncodeFixedI8(buf, pColData->info.type); tlen += taosEncodeFixedI32(buf, pColData->info.bytes); tlen += taosEncodeFixedBool(buf, pColData->hasNull); @@ -1614,7 +1614,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { for (int32_t i = 0; i < sz; i++) { SColumnInfoData data = {0}; buf = taosDecodeFixedI16(buf, &data.info.colId); - buf = taosDecodeFixedI16(buf, &data.info.type); + buf = taosDecodeFixedI8(buf, &data.info.type); buf = taosDecodeFixedI32(buf, &data.info.bytes); buf = taosDecodeFixedBool(buf, &data.hasNull); @@ -2074,6 +2074,10 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { // todo extract method + int32_t* version = (int32_t*)data; + *version = 1; + data += sizeof(int32_t); + int32_t* actualLen = (int32_t*)data; data += sizeof(int32_t); @@ -2085,8 +2089,11 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ *cols = numOfCols; data += sizeof(int32_t); - int32_t* hasColumnSegment = (int32_t*)data; - *hasColumnSegment = 1; + // flag segment. + // the inital bit is for column info + int32_t* flagSegment = (int32_t*)data; + *flagSegment = (1<<31); + data += sizeof(int32_t); uint64_t* groupId = (uint64_t*)data; @@ -2095,8 +2102,8 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - *((int16_t*)data) = pColInfoData->info.type; - data += sizeof(int16_t); + *((int8_t*)data) = pColInfoData->info.type; + data += sizeof(int8_t); *((int32_t*)data) = pColInfoData->info.bytes; data += sizeof(int32_t); @@ -2145,6 +2152,10 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { const char* pStart = pData; + int32_t version = *(int32_t*) pStart; + pStart += sizeof(int32_t); + ASSERT(version == 1); + // total length sizeof(int32_t) int32_t dataLen = *(int32_t*)pStart; pStart += sizeof(int32_t); @@ -2158,7 +2169,8 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(int32_t); // has column info segment - int32_t hasColumnInfo = *(int32_t*)pStart; + int32_t flagSeg = *(int32_t*)pStart; + int32_t hasColumnInfo = (flagSeg >> 31); pStart += sizeof(int32_t); // group id sizeof(uint64_t) @@ -2173,7 +2185,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData->info.type = *(int16_t*)pStart; - pStart += sizeof(int16_t); + pStart += sizeof(int8_t); pColInfoData->info.bytes = *(int32_t*)pStart; pStart += sizeof(int32_t); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 752b93bcea..b7e099d0b1 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4071,7 +4071,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else { ASSERT(0); } - pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; + + if (pOperator != NULL) { + pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; + } + return pOperator; } From 0aef734a20c412f3c6ff00a526dffc3ccebb020b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Aug 2022 17:56:26 +0800 Subject: [PATCH 050/165] fix(query): fix syntax error. --- source/libs/executor/src/executorimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 844ee4cc6e..5f5c035523 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1517,7 +1517,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); - doUpdateNumOfRows(pRow, numOfExprs, rowEntryOffset); + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); // no results, continue to check the next one if (pRow->numOfRows == 0) { From bf932e1d777331d388b28fc9c77a7d8460cb3784 Mon Sep 17 00:00:00 2001 From: huolibo Date: Mon, 8 Aug 2022 18:03:52 +0800 Subject: [PATCH 051/165] docs: add tmq sample code --- docs/zh/14-reference/03-connector/java.mdx | 121 ++++++++++++++++----- source/client/src/TSDBJNIConnector.c | 14 +-- 2 files changed, 103 insertions(+), 32 deletions(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index 88a3674671..d35e44030b 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -670,55 +670,126 @@ public class SchemalessInsertTest { TDengine Java 连接器支持订阅功能，应用 API 如下： -#### 创建订阅 +#### 创建 Topic ```java -TSDBSubscribe sub = ((TSDBConnection)conn).subscribe("topic", "select * from meters", false); +Connection connection = DriverManager.getConnection(url, properties); +Statement statement = connection.createStatement(); +statement.executeUpdate("create topic if not exists topic_speed as select ts, speed from speed_table"); ``` `subscribe` 方法的三个参数含义如下： -- topic：订阅的主题（即名称），此参数是订阅的唯一标识 -- sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据 -- restart：如果订阅已经存在，是重新开始，还是继续之前的订阅 +- topic_speed：订阅的主题（即名称），此参数是订阅的唯一标识。 +- sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据。 -如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic` 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。 +如上面的例子将使用 SQL 语句 `select ts, speed from speed_table` 创建一个名为 `topic_speed` 的订阅。 + +#### 创建 Consumer + +```java +Properties config = new Properties(); +config.setProperty("enable.auto.commit", "true"); +config.setProperty("group.id", "group1"); +config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer"); + +TaosConsumer consumer = new TaosConsumer<>(config); +``` + +- enable.auto.commit: 是否允许自动提交。 +- group.id: consumer: 所在的 group。 +- value.deserializer: 结果集反序列化方法，可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`，根据 SQL 的 resultSet 自定义反序列化方式。 #### 订阅消费数据 ```java -int total = 0; while(true) { - TSDBResultSet rs = sub.consume(); - int count = 0; - while(rs.next()) { - count++; - } - total += count; - System.out.printf("%d rows consumed, total %d\n", count, total); - Thread.sleep(1000); + ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); + for (ResultBean record : records) { + process(record); + } } ``` -`consume` 方法返回一个结果集，其中包含从上次 `consume` 到目前为止的所有新数据。请务必按需选择合理的调用 `consume` 的频率（如例子中的 `Thread.sleep(1000)`），否则会给服务端造成不必要的压力。 +`poll` 方法返回一个结果集，其中包含从上次 `poll` 到目前为止的所有新数据。请务必按需选择合理的调用 `poll` 的频率（如例子中的 `Duration.ofMillis(100)`），否则会给服务端造成不必要的压力。 #### 关闭订阅 ```java -sub.close(true); +consumer.close() ``` -`close` 方法关闭一个订阅。如果其参数为 `true` 表示保留订阅进度信息，后续可以创建同名订阅继续消费数据；如为 `false` 则不保留订阅进度。 - -### 关闭资源 +### 使用示例如下： ```java -resultSet.close(); -stmt.close(); -conn.close(); -``` +public abstract class ConsumerLoop { + private final TaosConsumer consumer; + private final List topics; + private final AtomicBoolean shutdown; + private final CountDownLatch shutdownLatch; -> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。 + public ConsumerLoop() throws SQLException { + Properties config = new Properties(); + config.setProperty("msg.with.table.name", "true"); + config.setProperty("enable.auto.commit", "true"); + config.setProperty("group.id", "group1"); + config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer"); + + this.consumer = new TaosConsumer<>(config); + this.topics = Collections.singletonList("topic_speed"); + this.shutdown = new AtomicBoolean(false); + this.shutdownLatch = new CountDownLatch(1); + } + + public abstract void process(ResultBean result); + + public void pollData() throws SQLException { + try { + consumer.subscribe(topics); + + while (!shutdown.get()) { + ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); + for (ResultBean record : records) { + process(record); + } + } + } finally { + consumer.close(); + shutdownLatch.countDown(); + } + } + + public void shutdown() throws InterruptedException { + shutdown.set(true); + shutdownLatch.await(); + } + + static class ResultDeserializer extends ReferenceDeserializer { + + } + + static class ResultBean { + private Timestamp ts; + private int speed; + + public Timestamp getTs() { + return ts; + } + + public void setTs(Timestamp ts) { + this.ts = ts; + } + + public int getSpeed() { + return speed; + } + + public void setSpeed(int speed) { + this.speed = speed; + } + } +} +``` ### 与连接池使用 diff --git a/source/client/src/TSDBJNIConnector.c b/source/client/src/TSDBJNIConnector.c index 227c2fff18..38bbec24ce 100644 --- a/source/client/src/TSDBJNIConnector.c +++ b/source/client/src/TSDBJNIConnector.c @@ -757,7 +757,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp(J int32_t code = taos_stmt_prepare(pStmt, str, len); taosMemoryFreeClear(str); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniError("prepareStmt jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; } @@ -785,7 +785,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameI if (code != TSDB_CODE_SUCCESS) { (*env)->ReleaseStringUTFChars(env, jname, name); - jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); + jniError("bindTableName jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); return JNI_TDENGINE_ERROR; } @@ -860,7 +860,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsI (*env)->ReleaseStringUTFChars(env, tableName, name); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); + jniError("tableNameTags jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); return JNI_TDENGINE_ERROR; } return JNI_SUCCESS; @@ -926,7 +926,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp( taosMemoryFreeClear(b); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniError("bindColData jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; } @@ -949,7 +949,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_addBatchImp(JNIEn int32_t code = taos_stmt_add_batch(pStmt); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniError("add batch jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; } @@ -973,7 +973,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(J int32_t code = taos_stmt_execute(pStmt); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniError("excute batch jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; } @@ -997,7 +997,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv int32_t code = taos_stmt_close(pStmt); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniError("close stmt jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); return JNI_TDENGINE_ERROR; } From c07097ab3f9107f7a11354572a262fd8e0790ec7 Mon Sep 17 00:00:00 2001 From: huolibo Date: Mon, 8 Aug 2022 18:43:57 +0800 Subject: [PATCH 052/165] docs: add reference --- docs/zh/14-reference/03-connector/java.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index d35e44030b..38c250b110 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -699,6 +699,7 @@ TaosConsumer consumer = new TaosConsumer<>(config); - enable.auto.commit: 是否允许自动提交。 - group.id: consumer: 所在的 group。 - value.deserializer: 结果集反序列化方法，可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`，根据 SQL 的 resultSet 自定义反序列化方式。 +- 其他参数请参考：[Consumer 参数列表](/develop/tmq#创建-consumer-以及consumer-group) #### 订阅消费数据 From 1d274e581ad098065989e0c601b3e19a5bcaf79b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 9 Aug 2022 09:00:11 +0800 Subject: [PATCH 053/165] fix tag filter --- source/libs/index/src/indexFilter.c | 43 +++++++++++++++-------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 6dfdbf6840..96a42928d4 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -395,26 +395,29 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP param.val = buf; } } else { - if (left->colValType == TSDB_DATA_TYPE_FLOAT) { - if (right->colValType == TSDB_DATA_TYPE_DOUBLE) { - f = GET_DOUBLE_VAL(right->condValue); - param.val = &f; - } else if (right->colValType == TSDB_DATA_TYPE_BIGINT) { - f = *(int64_t *)(right->condValue); - param.val = &f; - } else { - f = *(int32_t *)(right->condValue); - param.val = &f; - } - } else if (left->colValType == TSDB_DATA_TYPE_DOUBLE) { - if (right->colValType == TSDB_DATA_TYPE_DOUBLE) { - d = GET_DOUBLE_VAL(right->condValue); - param.val = &d; - } else if (right->colValType == TSDB_DATA_TYPE_BIGINT) { - d = *(int64_t *)(right->condValue); - param.val = &d; - } - } + // int8_t i8; int16_t i16; int32_t i32, uint8_t u8; uint16_t u16; uint32_t u32; + // if (right->colValType == TSDB_DATA_TYPE_TINYINT) { + // } + // if (left->colValType == TSDB_DATA_TYPE_FLOAT) { + // if (right->colValType == TSDB_DATA_TYPE_DOUBLE) { + // f = GET_DOUBLE_VAL(right->condValue); + // param.val = &f; + // } else if (right->colValType == TSDB_DATA_TYPE_BIGINT) { + // f = *(int64_t *)(right->condValue); + // param.val = &f; + // } else { + // f = *(int32_t *)(right->condValue); + // param.val = &f; + // } + // } else if (left->colValType == TSDB_DATA_TYPE_DOUBLE) { + // if (right->colValType == TSDB_DATA_TYPE_DOUBLE) { + // d = GET_DOUBLE_VAL(right->condValue); + // param.val = &d; + // } else if (right->colValType == TSDB_DATA_TYPE_BIGINT) { + // d = *(int64_t *)(right->condValue); + // param.val = &d; + // } + // } } ret = metaFilterTableIds(arg->metaEx, ¶m, output->result); } From 7e1917f105539a5e89563c01a72dea893e5d5d1b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 09:55:03 +0800 Subject: [PATCH 054/165] enh: refine the data requirements of the partition physical plan --- source/libs/function/src/builtins.c | 12 ++++++------ source/libs/planner/src/planOptimizer.c | 4 ++-- source/libs/planner/src/planUtil.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 13ebcd4518..9102137c31 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2452,7 +2452,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "histogram", .type = FUNCTION_TYPE_HISTOGRAM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateHistogram, .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, @@ -2467,7 +2467,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_histogram_partial", .type = FUNCTION_TYPE_HISTOGRAM_PARTIAL, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateHistogramPartial, .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, @@ -2479,7 +2479,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_histogram_merge", .type = FUNCTION_TYPE_HISTOGRAM_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateHistogramMerge, .getEnvFunc = getHistogramFuncEnv, .initFunc = functionSetup, @@ -2491,7 +2491,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "hyperloglog", .type = FUNCTION_TYPE_HYPERLOGLOG, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateHLL, .getEnvFunc = getHLLFuncEnv, .initFunc = functionSetup, @@ -2505,7 +2505,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "_hyperloglog_partial", - .type = FUNCTION_TYPE_HYPERLOGLOG_PARTIAL | FUNC_MGT_TIMELINE_FUNC, + .type = FUNCTION_TYPE_HYPERLOGLOG_PARTIAL, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateHLLPartial, .getEnvFunc = getHLLFuncEnv, @@ -2517,7 +2517,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "_hyperloglog_merge", - .type = FUNCTION_TYPE_HYPERLOGLOG_MERGE | FUNC_MGT_TIMELINE_FUNC, + .type = FUNCTION_TYPE_HYPERLOGLOG_MERGE, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateHLLMerge, .getEnvFunc = getHLLFuncEnv, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 86a4441cff..0e83051538 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1711,7 +1711,7 @@ static bool eliminateProjOptCanChildConditionUseChildTargets(SLogicNode* pChild, if (!cxt.canUse) return false; } if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pChild) && NULL != ((SJoinLogicNode*)pChild)->pOnConditions) { - SJoinLogicNode* pJoinLogicNode = (SJoinLogicNode*)pChild; + SJoinLogicNode* pJoinLogicNode = (SJoinLogicNode*)pChild; CheckNewChildTargetsCxt cxt = {.pNewChildTargets = pNewChildTargets, .canUse = false}; nodesWalkExpr(pJoinLogicNode->pOnConditions, eliminateProjOptCanUseNewChildTargetsImpl, &cxt); if (!cxt.canUse) return false; @@ -1768,7 +1768,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pProjectNode->node.pChildren); nodesDestroyNode((SNode*)pProjectNode); - //if pChild is a project logic node, remove its projection which is not reference by its target. + // if pChild is a project logic node, remove its projection which is not reference by its target. alignProjectionWithTarget(pChild); } pCxt->optimized = true; diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index d7310892de..ac1589bec0 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -232,7 +232,8 @@ static int32_t adjustPartitionDataRequirement(SPartitionLogicNode* pPart, EDataO return TSDB_CODE_PLAN_INTERNAL_ERROR; } pPart->node.resultDataOrder = requirement; - pPart->node.requireDataOrder = requirement; + pPart->node.requireDataOrder = + (requirement >= DATA_ORDER_LEVEL_IN_BLOCK ? DATA_ORDER_LEVEL_GLOBAL : DATA_ORDER_LEVEL_NONE); return TSDB_CODE_SUCCESS; } From 30a64bdcd9315dedbd69c0680ec38e9a5b508273 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 10:16:00 +0800 Subject: [PATCH 055/165] fix(query): set correct schema. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 139 ++++++++++++------------- 1 file changed, 68 insertions(+), 71 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0749d7b133..e1e236bf11 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -14,6 +14,7 @@ */ #include "tsdb.h" +#include "osDef.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) typedef enum { @@ -129,7 +130,8 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; - STSchema* pSchema; + STSchema* pSchema;// the newest version schema + STSchema* pMemSchema;// the previous schema for in-memory data, to avoid load schema too many times SDataFReader* pFileReader; SVersionRange verRange; @@ -145,11 +147,10 @@ static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanI SRowMerger* pMerger); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); -static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow); +static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); static int32_t doAppendRowFromBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); -static void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); static void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, @@ -1230,6 +1231,31 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB return false; } +static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { + // always set the newest schema version in pReader->pSchema + if (pReader->pSchema == NULL) { + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1); + } + + if (sversion == pReader->pSchema->version) { + return pReader->pSchema; + } + + if (pReader->pMemSchema == NULL) { + int32_t code = + metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); + return pReader->pMemSchema; + } + + if (pReader->pMemSchema->version == sversion) { + return pReader->pMemSchema; + } + + taosMemoryFree(pReader->pMemSchema); + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); + return pReader->pMemSchema; +} + static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key) { SRowMerger merge = {0}; @@ -1241,6 +1267,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SArray* pDelList = pBlockScanInfo->delSkyline; bool freeTSRow = false; + uint64_t uid = pBlockScanInfo->uid; // ascending order traverse if (ASCENDING_TRAVERSE(pReader->order)) { @@ -1276,9 +1303,9 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerGetRow(&merge, &pTSRow); } } else { // descending order: mem rows -----> imem rows ------> file block - updateSchema(pRow, pBlockScanInfo->uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pReader->pSchema); + tRowMergerInit(&merge, pRow, pSchema); doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, &fRow); @@ -1289,7 +1316,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } tRowMergerClear(&merge); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; @@ -1333,7 +1360,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* } tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); return TSDB_CODE_SUCCESS; } else { // key > ik.ts || key > k.ts ASSERT(key != ik.ts); @@ -1342,7 +1369,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [4] ik.ts < k.ts <= key if (ik.ts < k.ts) { doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -1353,7 +1380,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [6] k.ts < ik.ts <= key if (k.ts < ik.ts) { doMergeMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -1365,7 +1392,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* ASSERT(key > ik.ts && key > k.ts); doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } @@ -1373,9 +1400,9 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { // descending order scan // [1/2] k.ts >= ik.ts && k.ts >= key if (k.ts >= ik.ts && k.ts >= key) { - updateSchema(pRow, uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pReader->pSchema); + tRowMergerInit(&merge, pRow, pSchema); doMergeRowsInBuf(&pBlockScanInfo->iter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader); if (ik.ts == k.ts) { @@ -1390,7 +1417,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* } tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); return TSDB_CODE_SUCCESS; } else { ASSERT(ik.ts != k.ts); // this case has been included in the previous if branch @@ -1399,7 +1426,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* // [4] ik.ts > key >= k.ts if (ik.ts > key) { doMergeMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -1414,7 +1441,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; } @@ -1427,7 +1454,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); taosMemoryFree(pTSRow); return TSDB_CODE_SUCCESS; @@ -1495,7 +1522,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); taosMemoryFree(pTSRow); tRowMergerClear(&merge); @@ -2146,6 +2173,7 @@ TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pRea } } + int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader) { while (1) { @@ -2166,22 +2194,8 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe break; } - int32_t sversion = TSDBROW_SVERSION(pRow); - STSchema* pTSchema = NULL; - if (pReader->pSchema == NULL || sversion != pReader->pSchema->version) { - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pTSchema); - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } - } else { - pTSchema = pReader->pSchema; - } - + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); tRowMergerAdd(pMerger, pRow, pTSchema); - - if (pTSchema != pReader->pSchema) { - taosMemoryFree(pTSchema); - } } return TSDB_CODE_SUCCESS; @@ -2289,21 +2303,13 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc return TSDB_CODE_SUCCESS; } -void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader) { - int32_t sversion = TSDBROW_SVERSION(pRow); - - if (pReader->pSchema == NULL) { - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pSchema); - } else if (pReader->pSchema->version != sversion) { - taosMemoryFreeClear(pReader->pSchema); - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pSchema); - } -} - void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow) { + + TSDBROW* pNextRow = NULL; + TSDBROW current = *pRow; + { // if the timestamp of the next valid row has a different ts, return current row directly - TSDBROW current = *pRow; pIter->hasVal = tsdbTbDataIterNext(pIter->iter); if (!pIter->hasVal) { @@ -2311,14 +2317,14 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe *freeTSRow = false; return; } else { // has next point in mem/imem - TSDBROW* pNextRow = getValidRow(pIter, pDelList, pReader); + pNextRow = getValidRow(pIter, pDelList, pReader); if (pNextRow == NULL) { *pTSRow = current.pTSRow; *freeTSRow = false; return; } - if (TSDBROW_KEY(¤t).ts != TSDBROW_KEY(pNextRow).ts) { + if (current.pTSRow->ts != pNextRow->pTSRow->ts) { *pTSRow = current.pTSRow; *freeTSRow = false; return; @@ -2327,30 +2333,20 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe } SRowMerger merge = {0}; - TSDBKEY k = TSDBROW_KEY(pRow); // get the correct schema for data in memory - int32_t sversion = TSDBROW_SVERSION(pRow); - STSchema* pTSchema = NULL; - if (pReader->pSchema == NULL || sversion != pReader->pSchema->version) { - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pTSchema); - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } - } else { - pTSchema = pReader->pSchema; - } + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); - tRowMergerInit2(&merge, pReader->pSchema, pRow, pTSchema); - doMergeRowsInBuf(pIter, uid, k.ts, pDelList, &merge, pReader); + tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + + STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + tRowMergerAdd(&merge, pNextRow, pTSchema1); + + doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); tRowMergerGetRow(&merge, pTSRow); tRowMergerClear(&merge); *freeTSRow = true; - - if (sversion != pReader->pSchema->version) { - taosMemoryFree(pTSchema); - } } void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, @@ -2361,17 +2357,17 @@ void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlo TSDBKEY ik = TSDBROW_KEY(piRow); if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem - updateSchema(piRow, pBlockScanInfo->uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pReader->pSchema); + tRowMergerInit(&merge, piRow, pSchema); doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, pRow); doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } else { - updateSchema(pRow, pBlockScanInfo->uid, pReader); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pReader->pSchema); + tRowMergerInit(&merge, pRow, pSchema); doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, piRow); @@ -2432,12 +2428,12 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow) { +int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid) { int32_t numOfRows = pBlock->info.rows; int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - STSchema* pSchema = pReader->pSchema; + STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); SColVal colVal = {0}; int32_t i = 0, j = 0; @@ -2453,7 +2449,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* col_id_t colId = pColInfoData->info.colId; if (colId == pSchema->columns[j].colId) { - tTSRowGetVal(pTSRow, pReader->pSchema, j, &colVal); + tTSRowGetVal(pTSRow, pSchema, j, &colVal); doCopyColVal(pColInfoData, numOfRows, i, &colVal, pSupInfo); i += 1; j += 1; @@ -2529,7 +2525,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e break; } - doAppendRowFromTSRow(pBlock, pReader, pTSRow); + doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo->uid); if (freeTSRow) { taosMemoryFree(pTSRow); } @@ -2755,6 +2751,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); + taosMemoryFree(pReader->pMemSchema); taosMemoryFreeClear(pReader); } From f13215a9016adc902b97a7ff0b9f2e05af9623f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 10:17:48 +0800 Subject: [PATCH 056/165] test:fix invalid test cases. --- tests/script/tsim/parser/insert_multiTbl.sim | 2 +- tests/script/tsim/parser/last_cache_query.sim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/tsim/parser/insert_multiTbl.sim b/tests/script/tsim/parser/insert_multiTbl.sim index 78d3352378..fb6da1cfba 100644 --- a/tests/script/tsim/parser/insert_multiTbl.sim +++ b/tests/script/tsim/parser/insert_multiTbl.sim @@ -49,7 +49,7 @@ if $rows != 12 then return -1 endi if $data02 != NULL then - print "expect: NULL, act: $data03" + print "expect: NULL, act: $data02" return -1 endi if $data03 != @binary00@ then diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim index 7bafe82f5d..ebbb784da5 100644 --- a/tests/script/tsim/parser/last_cache_query.sim +++ b/tests/script/tsim/parser/last_cache_query.sim @@ -304,8 +304,8 @@ endi if $data21 != 24 then return -1 endi -if $data22 != 11.000000000 then - print $data02 +if $data22 != 11.000000000 then + print expect 11.000000000 actual: $data22 return -1 endi if $data23 != 25 then From 29c84833d3d7916d1ef83418e7fb1931d7b537e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 10:34:01 +0800 Subject: [PATCH 057/165] fix(query):fix invalid free. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index e1e236bf11..714af80e83 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1279,6 +1279,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } else if (k.ts < key) { // k.ts < key doMergeMultiRows(pRow, pBlockScanInfo->uid, pIter, pDelList, &pTSRow, pReader, &freeTSRow); @@ -1290,6 +1291,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } else { // descending order scan if (key < k.ts) { @@ -1301,6 +1303,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } else { // descending order: mem rows -----> imem rows ------> file block STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); @@ -1312,13 +1315,17 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); tRowMergerGetRow(&merge, &pTSRow); + freeTSRow = true; } } tRowMergerClear(&merge); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - taosMemoryFree(pTSRow); + if (freeTSRow) { + taosMemoryFree(pTSRow); + } + return TSDB_CODE_SUCCESS; } From 43a67f6da9b8c990a03cbb93be5f20fca7f15487 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 10:57:44 +0800 Subject: [PATCH 058/165] fix: insert error of unbound columns --- source/libs/parser/src/parInsert.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 6e47e26a7a..4756f65bf5 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -18,10 +18,10 @@ #include "parInt.h" #include "parToken.h" #include "parUtil.h" +#include "query.h" #include "tglobal.h" #include "ttime.h" #include "ttypes.h" -#include "query.h" #define NEXT_TOKEN(pSql, sToken) \ do { \ @@ -1189,12 +1189,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, if (!isParseBindParam) { // set the null value for the columns that do not assign values if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(schema[i].type), true, schema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } *gotRow = true; @@ -1267,7 +1262,8 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) { - return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than INT32_MAX"); + return buildInvalidOperationMsg(&pCxt->msg, + "too many rows in sql, total number of rows should be less than INT32_MAX"); } dataBuf->numOfTables = 1; @@ -1339,7 +1335,8 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STa SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) { - return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than INT32_MAX"); + return buildInvalidOperationMsg(&pCxt->msg, + "too many rows in sql, total number of rows should be less than INT32_MAX"); } dataBuf->numOfTables = 1; @@ -2060,7 +2057,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu } } } - if(rowEnd) { + if (rowEnd) { tdSRowEnd(pBuilder); } #ifdef TD_DEBUG_PRINT_ROW @@ -2077,7 +2074,8 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { - return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than INT32_MAX"); + return buildInvalidOperationMsg(&pBuf, + "too many rows in sql, total number of rows should be less than INT32_MAX"); } } From 38f3cc3e78642e9873b20a4e8720c5ebb2606df3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 9 Aug 2022 11:16:40 +0800 Subject: [PATCH 059/165] fix(query): interp fill(linear) after data range do not fill --- source/libs/executor/src/timewindowoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0fec1e61d2..df5cc3324f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2542,7 +2542,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // check if need to interpolate after ts range // except for fill(next) - while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT) { + while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); From 4503a37b711bffeeb2c9ad2423f44da362ae66d3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 9 Aug 2022 11:54:04 +0800 Subject: [PATCH 060/165] enh: adjust the default value of supportVnodes parameters --- source/common/src/tglobal.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e01b2ea58e..8d3deff495 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -355,7 +355,11 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1; if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1; + + tsNumOfSupportVnodes = tsNumOfCores * 2; + tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; From 01b4e8bb97de4bc7dc1b901ce59dccec27968bf8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 9 Aug 2022 13:35:01 +0800 Subject: [PATCH 061/165] test: set default supportVnodes value --- tests/pytest/util/dnodes-random-fail.py | 3 --- tests/pytest/util/dnodes.py | 5 ++--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/pytest/util/dnodes-random-fail.py b/tests/pytest/util/dnodes-random-fail.py index 7cadca64a3..9b653844f8 100644 --- a/tests/pytest/util/dnodes-random-fail.py +++ b/tests/pytest/util/dnodes-random-fail.py @@ -24,12 +24,9 @@ class TDSimClient: self.cfgDict = { "numOfLogLines": "100000000", - "numOfThreadsPerCore": "2.0", "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "anyIp": "0", - "sdbDebugFlag": "135", "rpcDebugFlag": "135", "tmrDebugFlag": "131", "cDebugFlag": "135", diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 20e4e4abe6..e530695d1e 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -30,7 +30,6 @@ class TDSimClient: self.path = path self.cfgDict = { "numOfLogLines": "100000000", - "numOfThreadsPerCore": "2.0", "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", @@ -40,6 +39,7 @@ class TDSimClient: "udebugFlag": "143", "jnidebugFlag": "143", "qdebugFlag": "143", + "supportVnodes": "1024", "telemetryReporting": "0", } @@ -117,8 +117,6 @@ class TDDnode: self.valgrind = 0 self.remoteIP = "" self.cfgDict = { - "walLevel": "2", - "fsync": "1000", "monitor": "0", "maxShellConns": "30000", "locale": "en_US.UTF-8", @@ -139,6 +137,7 @@ class TDDnode: "qdebugFlag": "143", "numOfLogLines": "100000000", "statusInterval": "1", + "supportVnodes": "1024", "telemetryReporting": "0" } From a2a1da06dd056b58d040fb83957a239377b9520b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 13:45:44 +0800 Subject: [PATCH 062/165] other:merge 3.0 --- contrib/CMakeLists.txt | 17 +++++++ include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 57 ++++++++++++++--------- source/dnode/vnode/src/tq/tqExec.c | 13 +++++- source/libs/executor/src/dataDispatcher.c | 8 ++-- source/libs/executor/src/executor.c | 33 +++++++++---- source/libs/qworker/src/qworker.c | 29 ++++++------ source/libs/stream/src/streamExec.c | 23 ++++++--- 8 files changed, 124 insertions(+), 58 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b4e8825431..de7b75a245 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -27,6 +27,10 @@ else () cat("${TD_SUPPORT_DIR}/taosadapter_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() +if(TD_LINUX_64 AND JEMALLOC_ENABLED) + cat("${TD_SUPPORT_DIR}/jemalloc_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + # pthread if(${BUILD_PTHREAD}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -392,6 +396,19 @@ if(${BUILD_WITH_SQLITE}) endif(NOT TD_WINDOWS) endif(${BUILD_WITH_SQLITE}) +# jemalloc +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + include(ExternalProject) + ExternalProject_Add(jemalloc + PREFIX "jemalloc" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ + BUILD_COMMAND ${MAKE} + ) + INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include) +ENDIF () + # addr2line if(${BUILD_ADDR2LINE}) if(NOT ${TD_WINDOWS}) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 7bc2ede4fc..a3cc7aaa8a 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -123,7 +123,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table * @param handle * @return */ -int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds); +int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds); /** * kill the ongoing query and free the query handle and corresponding resources automatically diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 662558529d..ef9a64e946 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -604,28 +604,44 @@ _end: static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, int8_t blkType) { - while (1) { - SSDataBlock *output = NULL; - uint64_t ts; + SArray *pResList = taosArrayInit(1, POINTER_BYTES); + if (pResList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } - int32_t code = qExecTask(taskInfo, &output, &ts); + while (1) { + uint64_t ts; + int32_t code = qExecTask(taskInfo, pResList, &ts); if (code < 0) { smaError("vgId:%d, qExecTask for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr(code)); goto _err; } - if (output) { -#if 0 + if (taosArrayGetSize(pResList) == 0) { + if (terrno == 0) { + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); + } else { + smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, terrstr()); + goto _err; + } + + break; + } + + for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { + SSDataBlock *output = taosArrayGetP(pResList, i); + +#if 1 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, pItem->level); - SArray *pResult = taosArrayInit(1, sizeof(SSDataBlock)); - taosArrayPush(pResult, output); - blockDebugShowDataBlocks(pResult, flag); - taosArrayDestroy(pResult); +// blockDebugShowDataBlocks(output, flag); +// taosArrayDestroy(pResult); #endif - STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); + STsdb * sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); SSubmitReq *pReq = NULL; + // TODO: the schema update should be handled later(TD-17965) if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, SMA_VID(pSma), suid) < 0) { smaError("vgId:%d, build submit req for rsma stable %" PRIi64 " level %" PRIi8 " failed since %s", @@ -644,17 +660,14 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm SMA_VID(pSma), suid, pItem->level, output->info.version); taosMemoryFreeClear(pReq); - } else if (terrno == 0) { - smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); - break; - } else { - smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, terrstr()); - goto _err; } } + taosArrayDestroy(pResList); return TSDB_CODE_SUCCESS; + _err: + taosArrayDestroy(pResList); return TSDB_CODE_FAILED; } @@ -1407,7 +1420,7 @@ int32_t tdRSmaFetchSend(SSma *pSma, SRSmaInfo *pInfo, int8_t level) { tEncoderClear(&encoder); } tEncoderClear(&encoder); - + ((SMsgHead *)pBuf)->vgId = SMA_VID(pSma); ((SMsgHead *)pBuf)->contLen = contLen + sizeof(SMsgHead); @@ -1434,10 +1447,10 @@ _err: /** * @brief fetch rsma data of level 2/3 and submit - * - * @param pSma - * @param pMsg - * @return int32_t + * + * @param pSma + * @param pMsg + * @return int32_t */ int32_t smaProcessFetch(SSma *pSma, void *pMsg) { SRpcMsg *pRpcMsg = (SRpcMsg *)pMsg; diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 435bbb77b8..09a5f1a8c6 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -80,14 +80,22 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa } int32_t rowCnt = 0; + SArray* pResList = taosArrayInit(4, POINTER_BYTES); + while (1) { + taosArrayClear(pResList); + SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; tqDebug("task start to execute"); - if (qExecTask(task, &pDataBlock, &ts) < 0) { + if (qExecTask(task, pResList, &ts) < 0) { ASSERT(0); } - tqDebug("task execute end, get %p", pDataBlock); + + if (taosArrayGetSize(pResList) > 0) { + pDataBlock = taosArrayGet(pResList, 0); + tqDebug("task execute end, get %p", pDataBlock); + } if (pDataBlock != NULL) { if (pRsp->withTbName) { @@ -143,6 +151,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa break; } + taosArrayDestroy(pResList); return 0; } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 3a37b5e760..977adcaa5e 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -69,10 +69,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { // clang-format off // data format: -// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | total length | numOfRows | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) -// | |sizeof(int32) | sizeof(int32)| sizeof(uint64_t) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | -// +----------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// +----------------+------------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// |SDataCacheEntry | version | total length | numOfRows | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) +// | | sizeof(int32_t) |sizeof(int32) | sizeof(int32)| sizeof(uint64_t) | (sizeof(int8_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | +// +----------------+------------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header // clang-format on diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 897ddb59ec..5bbf5928da 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -422,11 +422,17 @@ int waitMoment(SQInfo* pQInfo) { } #endif -int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { +static void freeBlock(void* param) { + SSDataBlock* pBlock = *(SSDataBlock**) param; + blockDataDestroy(pBlock); +} + +int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); - *pRes = NULL; + taosArrayClearEx(pResList, freeBlock); + int64_t curOwner = 0; if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); @@ -457,23 +463,34 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); + int32_t current = 0; + SSDataBlock* pRes = NULL; + int64_t st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); + while((pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot)) != NULL) { + SSDataBlock* p = createOneDataBlock(pRes, true); + current += p->info.rows; + ASSERT(p->info.rows > 0); + taosArrayPush(pResList, &p); + + if (current >= 4096) { + break; + } + } + uint64_t el = (taosGetTimestampUs() - st); pTaskInfo->cost.elapsedTime += el; - if (NULL == *pRes) { + if (NULL == pRes) { *useconds = pTaskInfo->cost.elapsedTime; } cleanUpUdfs(); - - int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; - qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", - GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0); + qDebug("%s task suspended, %d rows in %d blocks returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", + GET_TASKID(pTaskInfo), current, (int32_t) taosArrayGetSize(pResList), total, 0, el / 1000.0); atomic_store_64(&pTaskInfo->owner, 0); return pTaskInfo->code; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 7c5cfda81f..85a1b0444c 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -75,22 +75,20 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { int32_t code = 0; bool qcontinue = true; - SSDataBlock *pRes = NULL; uint64_t useconds = 0; int32_t i = 0; int32_t execNum = 0; qTaskInfo_t taskHandle = ctx->taskHandle; DataSinkHandle sinkHandle = ctx->sinkHandle; + SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (true) { QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); - pRes = NULL; - // if *taskHandle is NULL, it's killed right now if (taskHandle) { qwDbgSimulateSleep(); - code = qExecTask(taskHandle, &pRes, &useconds); + code = qExecTask(taskHandle, pResList, &useconds); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -103,9 +101,8 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { ++execNum; - if (NULL == pRes) { + if (taosArrayGetSize(pResList) == 0) { QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); - dsEndPut(sinkHandle, useconds); QW_ERR_RET(qwHandleTaskComplete(QW_FPARAMS(), ctx)); @@ -117,19 +114,20 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { break; } - int32_t rows = pRes->info.rows; + for(int32_t j = 0; j < taosArrayGetSize(pResList); ++j) { + SSDataBlock *pRes = taosArrayGetP(pResList, j); + ASSERT(pRes->info.rows > 0); - ASSERT(pRes->info.rows > 0); + SInputData inputData = {.pData = pRes}; + code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); + if (code) { + QW_TASK_ELOG("dsPutDataBlock failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_RET(code); + } - SInputData inputData = {.pData = pRes}; - code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); - if (code) { - QW_TASK_ELOG("dsPutDataBlock failed, code:%x - %s", code, tstrerror(code)); - QW_ERR_RET(code); + QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); } - QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", rows, qcontinue); - if (!qcontinue) { if (queryStop) { *queryStop = true; @@ -151,6 +149,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { } } + taosArrayDestroy(pResList); QW_RET(code); } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 7512f792c1..b92f658de0 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -43,13 +43,17 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* } // exec + SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SSDataBlock* output = NULL; uint64_t ts = 0; - if (qExecTask(exec, &output, &ts) < 0) { + + taosArrayClear(pResList); + if (qExecTask(exec, pResList, &ts) < 0) { ASSERT(false); } - if (output == NULL) { + + if (taosArrayGetSize(pResList) == 0) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SSDataBlock block = {0}; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data; @@ -65,6 +69,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* break; } + output = taosArrayGetP(pResList, 0); if (output->info.type == STREAM_RETRIEVE) { if (streamBroadcastToChildren(pTask, output) < 0) { // TODO @@ -79,6 +84,8 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); } + + taosArrayDestroy(pResList); return 0; } @@ -98,6 +105,7 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { void* exec = pTask->exec.executor; + SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { @@ -107,14 +115,17 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { int32_t batchCnt = 0; while (1) { - SSDataBlock* output = NULL; - uint64_t ts = 0; - if (qExecTask(exec, &output, &ts) < 0) { + uint64_t ts = 0; + taosArrayClear(pResList); + if (qExecTask(exec, pResList, &ts) < 0) { ASSERT(0); } - if (output == NULL) break; + + if (taosArrayGetSize(pResList) == 0) break; SSDataBlock block = {0}; + SSDataBlock* output = taosArrayGetP(pResList, 0); + assignOneDataBlock(&block, output); block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); From b9c6e0cdf9db43c6b56b3cf7ec7757ea2ee79843 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 13:55:51 +0800 Subject: [PATCH 063/165] refactor: stream invoke previous APIs. --- include/libs/executor/executor.h | 3 +- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/tq/tqExec.c | 13 +----- source/libs/executor/src/executor.c | 59 +++++++++++++++++++++++++- source/libs/qworker/src/qworker.c | 2 +- source/libs/stream/src/streamExec.c | 23 +++------- 6 files changed, 70 insertions(+), 32 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index a3cc7aaa8a..a64815f14f 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -123,7 +123,8 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table * @param handle * @return */ -int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds); +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds); +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds); /** * kill the ongoing query and free the query handle and corresponding resources automatically diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ef9a64e946..6b513f0242 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -612,7 +612,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm while (1) { uint64_t ts; - int32_t code = qExecTask(taskInfo, pResList, &ts); + int32_t code = qExecTaskOpt(taskInfo, pResList, &ts); if (code < 0) { smaError("vgId:%d, qExecTask for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr(code)); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 09a5f1a8c6..435bbb77b8 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -80,22 +80,14 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa } int32_t rowCnt = 0; - SArray* pResList = taosArrayInit(4, POINTER_BYTES); - while (1) { - taosArrayClear(pResList); - SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; tqDebug("task start to execute"); - if (qExecTask(task, pResList, &ts) < 0) { + if (qExecTask(task, &pDataBlock, &ts) < 0) { ASSERT(0); } - - if (taosArrayGetSize(pResList) > 0) { - pDataBlock = taosArrayGet(pResList, 0); - tqDebug("task execute end, get %p", pDataBlock); - } + tqDebug("task execute end, get %p", pDataBlock); if (pDataBlock != NULL) { if (pRsp->withTbName) { @@ -151,7 +143,6 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa break; } - taosArrayDestroy(pResList); return 0; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 5bbf5928da..328a65bec4 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -427,7 +427,7 @@ static void freeBlock(void* param) { blockDataDestroy(pBlock); } -int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); @@ -496,6 +496,63 @@ int32_t qExecTask(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { return pTaskInfo->code; } +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + int64_t threadId = taosGetSelfPthreadId(); + + *pRes = NULL; + int64_t curOwner = 0; + if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { + qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); + pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; + return pTaskInfo->code; + } + + if (pTaskInfo->cost.start == 0) { + pTaskInfo->cost.start = taosGetTimestampMs(); + } + + if (isTaskKilled(pTaskInfo)) { + atomic_store_64(&pTaskInfo->owner, 0); + qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); + return TSDB_CODE_SUCCESS; + } + + // error occurs, record the error code and return to client + int32_t ret = setjmp(pTaskInfo->env); + if (ret != TSDB_CODE_SUCCESS) { + pTaskInfo->code = ret; + cleanUpUdfs(); + qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); + atomic_store_64(&pTaskInfo->owner, 0); + + return pTaskInfo->code; + } + + qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); + + int64_t st = taosGetTimestampUs(); + + *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); + uint64_t el = (taosGetTimestampUs() - st); + + pTaskInfo->cost.elapsedTime += el; + if (NULL == *pRes) { + *useconds = pTaskInfo->cost.elapsedTime; + } + + cleanUpUdfs(); + + int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; + uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; + + qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", + GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0); + + atomic_store_64(&pTaskInfo->owner, 0); + return pTaskInfo->code; +} + int32_t qKillTask(qTaskInfo_t qinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; if (pTaskInfo == NULL) { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 85a1b0444c..e06b752862 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -88,7 +88,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { // if *taskHandle is NULL, it's killed right now if (taskHandle) { qwDbgSimulateSleep(); - code = qExecTask(taskHandle, pResList, &useconds); + code = qExecTaskOpt(taskHandle, pResList, &useconds); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index b92f658de0..7512f792c1 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -43,17 +43,13 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* } // exec - SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SSDataBlock* output = NULL; uint64_t ts = 0; - - taosArrayClear(pResList); - if (qExecTask(exec, pResList, &ts) < 0) { + if (qExecTask(exec, &output, &ts) < 0) { ASSERT(false); } - - if (taosArrayGetSize(pResList) == 0) { + if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SSDataBlock block = {0}; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data; @@ -69,7 +65,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* break; } - output = taosArrayGetP(pResList, 0); if (output->info.type == STREAM_RETRIEVE) { if (streamBroadcastToChildren(pTask, output) < 0) { // TODO @@ -84,8 +79,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); } - - taosArrayDestroy(pResList); return 0; } @@ -105,7 +98,6 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { void* exec = pTask->exec.executor; - SArray* pResList = taosArrayInit(4, POINTER_BYTES); while (1) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { @@ -115,17 +107,14 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { int32_t batchCnt = 0; while (1) { - uint64_t ts = 0; - taosArrayClear(pResList); - if (qExecTask(exec, pResList, &ts) < 0) { + SSDataBlock* output = NULL; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { ASSERT(0); } - - if (taosArrayGetSize(pResList) == 0) break; + if (output == NULL) break; SSDataBlock block = {0}; - SSDataBlock* output = taosArrayGetP(pResList, 0); - assignOneDataBlock(&block, output); block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); From aaa588a62f1fd031c00807192c2bc282a200ae3f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 9 Aug 2022 14:10:14 +0800 Subject: [PATCH 064/165] fix: fix stmt parser crash issue --- source/libs/catalog/src/ctgDbg.c | 2 +- source/libs/parser/src/parInsert.c | 7 ++++ source/libs/scheduler/inc/schInt.h | 9 ++++- source/libs/scheduler/src/schFlowCtrl.c | 4 +- source/libs/scheduler/src/schTask.c | 53 ++++++++++++++++++++----- source/util/src/terror.c | 2 +- 6 files changed, 63 insertions(+), 14 deletions(-) diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 8333cb28c0..bd3402dc39 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.lockEnable = true}; +SCtgDebug gCTGDebug = {0}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 85f73f0663..fa86cfb5b5 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1537,6 +1537,13 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache if (pContext->pStmtCb && *pQuery) { (*pContext->pStmtCb->getExecInfoFn)(pContext->pStmtCb->pStmt, &context.pVgroupsHashObj, &context.pTableBlockHashObj); + if (NULL == context.pVgroupsHashObj) { + context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + } + if (NULL == context.pTableBlockHashObj) { + context.pTableBlockHashObj = + taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } } else { context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); context.pTableBlockHashObj = diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 02e878f4f8..1b3d75f33b 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -61,6 +61,8 @@ typedef enum { #define SCH_MAX_TASK_TIMEOUT_USEC 60000000 #define SCH_DEFAULT_MAX_RETRY_NUM 6 +#define SCH_ASYNC_LAUNCH_TASK 0 + typedef struct SSchDebug { bool lockEnable; bool apiEnable; @@ -281,6 +283,11 @@ typedef struct SSchJob { SQueryProfileSummary summary; } SSchJob; +typedef struct SSchTaskCtx { + SSchJob *pJob; + SSchTask *pTask; +} SSchTaskCtx; + extern SSchedulerMgmt schMgmt; #define SCH_TASK_TIMEOUT(_task) ((taosGetTimestampUs() - *(int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId)) > (_task)->timeoutUsec) @@ -428,7 +435,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask); int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough); int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask); -int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask); +int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask); int32_t schLaunchFetchTask(SSchJob *pJob); int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode); int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction); diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 6b34a394b6..c5c2bfb2bb 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -54,7 +54,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { sum += pTask->plan->execNodeStat.tableNum; } - if (sum < schMgmt.cfg.maxNodeTableNum) { + if (schMgmt.cfg.maxNodeTableNum <= 0 || sum < schMgmt.cfg.maxNodeTableNum) { SCH_JOB_DLOG("job no need flow ctrl, totalTableNum:%d", sum); return TSDB_CODE_SUCCESS; } @@ -230,7 +230,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); - SCH_ERR_JRET(schLaunchTaskImpl(pJob, pTask)); + SCH_ERR_JRET(schAsyncLaunchTaskImpl(pJob, pTask)); remainNum -= pTask->plan->execNodeStat.tableNum; if (remainNum <= 0) { diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index cabca0dc0c..729dbf7c1f 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -819,7 +819,10 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { return TSDB_CODE_SUCCESS; } -int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { +int32_t schLaunchTaskImpl(void *param) { + SSchTaskCtx *pCtx = (SSchTaskCtx *)param; + SSchJob *pJob = pCtx->pJob; + SSchTask *pTask = pCtx->pTask; int8_t status = 0; int32_t code = 0; @@ -834,12 +837,12 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("no need to launch task cause of job status %s", jobTaskStatusStr(status)); - SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); + SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } // NOTE: race condition: the task should be put into the hash table before send msg to server if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXEC) { - SCH_ERR_RET(schPushTaskToExecList(pJob, pTask)); + SCH_ERR_JRET(schPushTaskToExecList(pJob, pTask)); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXEC); } @@ -850,19 +853,51 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { if (TSDB_CODE_SUCCESS != code) { SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, pTask->msgLen); - SCH_ERR_RET(code); + SCH_ERR_JRET(code); } else { SCH_TASK_DLOGL("physical plan len:%d, %s", pTask->msgLen, pTask->msg); } } - SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask)); + SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); if (SCH_IS_QUERY_JOB(pJob)) { - SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); + SCH_ERR_JRET(schEnsureHbConnection(pJob, pTask)); } - SCH_ERR_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); + SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); + +_return: + + taosMemoryFree(param); + +#if SCH_ASYNC_LAUNCH_TASK + if (code) { + code = schProcessOnTaskFailure(pJob, pTask, code); + } + if (code) { + code = schHandleJobFailure(pJob, code); + } +#endif + + SCH_RET(code); +} + +int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { + + SSchTaskCtx *param = taosMemoryCalloc(1, sizeof(SSchTaskCtx)); + if (NULL == param) { + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + param->pJob = pJob; + param->pTask = pTask; + +#if SCH_ASYNC_LAUNCH_TASK + taosAsyncExec(schLaunchTaskImpl, param, NULL); +#else + SCH_ERR_RET(schLaunchTaskImpl(param)); +#endif return TSDB_CODE_SUCCESS; } @@ -878,10 +913,10 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_JRET(schCheckIncTaskFlowQuota(pJob, pTask, &enough)); if (enough) { - SCH_ERR_JRET(schLaunchTaskImpl(pJob, pTask)); + SCH_ERR_JRET(schAsyncLaunchTaskImpl(pJob, pTask)); } } else { - SCH_ERR_JRET(schLaunchTaskImpl(pJob, pTask)); + SCH_ERR_JRET(schAsyncLaunchTaskImpl(pJob, pTask)); } return TSDB_CODE_SUCCESS; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3c31c893d1..2be1c9f744 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -47,7 +47,7 @@ STaosError errors[] = { // rpc TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, "Authentication failure") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, "Redirect") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, "Database not ready, need retry") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "Port already in use") From d9a4ea305dc093ae7f345f58b2b4bb3270ec118c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 9 Aug 2022 13:55:04 +0800 Subject: [PATCH 065/165] fix(stream): memory error --- include/libs/stream/tstream.h | 36 +++++++++++++++---------- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/vnode/src/tq/tq.c | 30 ++++++++++++--------- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 +++--- source/libs/stream/src/stream.c | 2 ++ source/libs/stream/src/streamExec.c | 7 +++-- source/libs/stream/src/streamRecover.c | 20 +++++++++----- 7 files changed, 66 insertions(+), 39 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index f5074becec..8aacfec397 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -42,8 +42,8 @@ enum { TASK_STATUS__DROPPING, TASK_STATUS__FAIL, TASK_STATUS__STOP, - TASK_STATUS__PREPARE_RECOVER, - TASK_STATUS__RECOVERING, + TASK_STATUS__RECOVER_DOWNSTREAM, + TASK_STATUS__RECOVER_SELF, }; enum { @@ -232,8 +232,8 @@ typedef struct { } SStreamChildEpInfo; typedef struct { - int32_t nodeId; - int32_t childId; + int32_t srcNodeId; + int32_t srcChildId; int64_t stateSaveVer; int64_t stateProcessedVer; } SStreamCheckpointInfo; @@ -394,9 +394,6 @@ typedef struct { int32_t upstreamTaskId; int32_t upstreamChildId; int32_t upstreamNodeId; -#if 0 - int64_t sourceVer; -#endif int32_t blockNum; SArray* dataLen; // SArray SArray* data; // SArray @@ -426,6 +423,7 @@ typedef struct { int32_t rspToTaskId; } SStreamRetrieveRsp; +#if 0 typedef struct { int64_t streamId; int32_t taskId; @@ -462,13 +460,25 @@ int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp); int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pRsp); -typedef struct { - int64_t streamId; -} SPStreamTaskRecoverReq; +int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg); +int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp); +#endif typedef struct { - int8_t reserved; -} SPStreamTaskRecoverRsp; + int64_t streamId; + int32_t downstreamTaskId; + int32_t taskId; +} SStreamRecoverDownstreamReq; + +typedef struct { + int64_t streamId; + int32_t downstreamTaskId; + int32_t taskId; + SArray* checkpointVer; // SArray +} SStreamRecoverDownstreamRsp; + +int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq); +int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, const SStreamRecoverDownstreamRsp* pRsp); int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); @@ -479,8 +489,6 @@ int32_t streamSetupTrigger(SStreamTask* pTask); int32_t streamProcessRunReq(SStreamTask* pTask); int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg, bool exec); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp); -int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg); -int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp); int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index ba9bb2982f..7bde05aca5 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -504,6 +504,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return 0; } +#if 0 static int32_t mndPersistTaskRecoverReq(STrans *pTrans, SStreamTask *pTask) { SMStreamTaskRecoverReq *pReq = taosMemoryCalloc(1, sizeof(SMStreamTaskRecoverReq)); if (pReq == NULL) { @@ -540,7 +541,6 @@ static int32_t mndPersistTaskRecoverReq(STrans *pTrans, SStreamTask *pTask) { return 0; } -#if 0 int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { if (pStream->isDistributed) { int32_t lv = taosArrayGetSize(pStream->tasks); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f7da287ca1..02ca248054 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -777,6 +777,7 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { } } +#if 0 int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; @@ -789,18 +790,6 @@ int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { } } -int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { - SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t taskId = pRsp->taskId; - SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); - if (pTask) { - streamProcessDispatchRsp(pTask, pRsp); - return 0; - } else { - return -1; - } -} - int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverRsp* pRsp = pMsg->pCont; int32_t taskId = pRsp->rspTaskId; @@ -813,6 +802,19 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { return -1; } } +#endif + +int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { + SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t taskId = pRsp->taskId; + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { + streamProcessDispatchRsp(pTask, pRsp); + return 0; + } else { + return -1; + } +} int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; @@ -873,6 +875,8 @@ void vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { .code = 0, }; streamProcessDispatchReq(pTask, &req, &rsp, false); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); return; } @@ -883,4 +887,6 @@ FAIL: .info = pMsg->info, }; tmsgSendRsp(&rsp); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fbbe5bc695..0cb66484c0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -332,14 +332,14 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { case TDMT_STREAM_TASK_DISPATCH: // return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, pInfo->workerId != 0); return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); - case TDMT_STREAM_TASK_RECOVER: - return tqProcessTaskRecoverReq(pVnode->pTq, pMsg); + /*case TDMT_STREAM_TASK_RECOVER:*/ + /*return tqProcessTaskRecoverReq(pVnode->pTq, pMsg);*/ case TDMT_STREAM_RETRIEVE: return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg); - case TDMT_STREAM_TASK_RECOVER_RSP: - return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg); + /*case TDMT_STREAM_TASK_RECOVER_RSP:*/ + /*return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg);*/ case TDMT_STREAM_RETRIEVE_RSP: return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg); default: diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index e4bf90f308..1f8d742de4 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -229,6 +229,7 @@ int32_t streamProcessRunReq(SStreamTask* pTask) { return 0; } +#if 0 int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pRsp) { void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamTaskRecoverRsp)); ((SMsgHead*)buf)->vgId = htonl(pReq->upstreamNodeId); @@ -267,6 +268,7 @@ int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) return 0; } +#endif int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { qDebug("task %d receive retrieve req from node %d task %d", pTask->taskId, pReq->srcNodeId, pReq->srcTaskId); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 7512f792c1..ffb7c04bf2 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -200,14 +200,13 @@ int32_t streamExecForAll(SStreamTask* pTask) { streamTaskExecImpl(pTask, data, pRes); qDebug("stream task %d exec end", pTask->taskId); - streamFreeQitem(data); - if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); if (qRes == NULL) { // TODO log failed ver streamQueueProcessFail(pTask->inputQueue); taosArrayDestroy(pRes); + streamFreeQitem(data); return -1; } qRes->type = STREAM_INPUT__DATA_BLOCK; @@ -224,10 +223,14 @@ int32_t streamExecForAll(SStreamTask* pTask) { /*streamQueueProcessFail(pTask->inputQueue);*/ taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); taosFreeQitem(qRes); + streamFreeQitem(data); return -1; } /*streamQueueProcessSuccess(pTask->inputQueue);*/ + } else { + taosArrayDestroy(pRes); } + streamFreeQitem(data); } return 0; } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 2b5af0684d..b2a7e00877 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -15,6 +15,7 @@ #include "streamInc.h" +#if 0 int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -86,17 +87,18 @@ int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp tEndDecode(pDecoder); return 0; } +#endif int32_t tEncodeSStreamCheckpointInfo(SEncoder* pEncoder, const SStreamCheckpointInfo* pCheckpoint) { - if (tEncodeI32(pEncoder, pCheckpoint->nodeId) < 0) return -1; - if (tEncodeI32(pEncoder, pCheckpoint->childId) < 0) return -1; + if (tEncodeI32(pEncoder, pCheckpoint->srcNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pCheckpoint->srcChildId) < 0) return -1; if (tEncodeI64(pEncoder, pCheckpoint->stateProcessedVer) < 0) return -1; return 0; } int32_t tDecodeSStreamCheckpointInfo(SDecoder* pDecoder, SStreamCheckpointInfo* pCheckpoint) { - if (tDecodeI32(pDecoder, &pCheckpoint->nodeId) < 0) return -1; - if (tDecodeI32(pDecoder, &pCheckpoint->childId) < 0) return -1; + if (tDecodeI32(pDecoder, &pCheckpoint->srcNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pCheckpoint->srcChildId) < 0) return -1; if (tDecodeI64(pDecoder, &pCheckpoint->stateProcessedVer) < 0) return -1; return 0; } @@ -221,11 +223,17 @@ int32_t streamSaveAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) { return 0; } -int32_t streamFetchSinkStatus(SStreamTask* pTask) { - ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); +int32_t streamFetchDownstreamStatus(SStreamTask* pTask) { // set self status to recover_phase1 // build fetch status msg // send fetch msg + atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_DOWNSTREAM); + + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else { + ASSERT(0); + } return 0; } From 7f7a2439ff1413486f2244d4bf29b04c0f4ed755 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 14:19:43 +0800 Subject: [PATCH 066/165] enh: adjust the content of the taos sql overview --- docs/zh/12-taos-sql/index.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/zh/12-taos-sql/index.md b/docs/zh/12-taos-sql/index.md index cb01b3a918..900fff1ba2 100644 --- a/docs/zh/12-taos-sql/index.md +++ b/docs/zh/12-taos-sql/index.md @@ -5,11 +5,12 @@ description: "TAOS SQL 支持的语法规则、主要查询功能、支持的 SQ 本文档说明 TAOS SQL 支持的语法规则、主要查询功能、支持的 SQL 查询函数，以及常用技巧等内容。阅读本文档需要读者具有基本的 SQL 语言的基础。 -TAOS SQL 是用户对 TDengine 进行数据写入和查询的主要工具。TAOS SQL 为了便于用户快速上手，在一定程度上提供与标准 SQL 类似的风格和模式。严格意义上，TAOS SQL 并不是也不试图提供标准的 SQL 语法。此外，由于 TDengine 针对的时序性结构化数据不提供删除功能，因此在 TAO SQL 中不提供数据删除的相关功能。 +TAOS SQL 是用户对 TDengine 进行数据写入和查询的主要工具。TAOS SQL 提供标准的 SQL 语法，并针对时序数据和业务的特点优化和新增了许多语法和功能。TAOS SQL 语句的最大长度为 1M。TAOS SQL 不支持关键字的缩写，例如 DELETE 不能缩写为 DEL。 本章节 SQL 语法遵循如下约定： -- <\> 里的内容是用户需要输入的，但不要输入 <\> 本身 +- 用大写字母表示关键字，但 SQL 本身并不区分关键字和标识符的大小写 +- 用小写字母表示需要用户输入的内容 - \[ \] 表示内容为可选项，但不能输入 [] 本身 - | 表示多选一，选择其中一个即可，但不能输入 | 本身 - … 表示前面的项可重复多个 From 17caf2c6291dbf25d6c7e86f2503f78aa9038d53 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 9 Aug 2022 14:24:27 +0800 Subject: [PATCH 067/165] doc: fix broken links --- docs/zh/14-reference/03-connector/java.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index 38c250b110..a7e8c1fa6e 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -358,7 +358,7 @@ JDBC 连接器可能报错的错误码包括 3 种：JDBC driver 本身的报错 具体的错误码请参考： - [TDengine Java Connector](https://github.com/taosdata/taos-connector-jdbc/blob/main/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java) -- [TDengine_ERROR_CODE](https://github.com/taosdata/TDengine/blob/develop/src/inc/taoserror.h) +- [TDengine_ERROR_CODE](../error-code) ### 通过参数绑定写入数据 @@ -699,7 +699,7 @@ TaosConsumer consumer = new TaosConsumer<>(config); - enable.auto.commit: 是否允许自动提交。 - group.id: consumer: 所在的 group。 - value.deserializer: 结果集反序列化方法，可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`，根据 SQL 的 resultSet 自定义反序列化方式。 -- 其他参数请参考：[Consumer 参数列表](/develop/tmq#创建-consumer-以及consumer-group) +- 其他参数请参考：[Consumer 参数列表](../../../develop/tmq#创建-consumer-以及consumer-group) #### 订阅消费数据 From 2731434053d5dd302c9f32d3b93619d8e08b0b5c Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 9 Aug 2022 14:25:44 +0800 Subject: [PATCH 068/165] doc: fix broken link --- docs/zh/14-reference/03-connector/java.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index a7e8c1fa6e..c45d670bc6 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -883,7 +883,7 @@ Query OK, 1 row(s) in set (0.000141s) - SpringJdbcTemplate：Spring JdbcTemplate 中使用 taos-jdbcdriver。 - mybatisplus-demo：Springboot + Mybatis 中使用 taos-jdbcdriver。 -请参考：[JDBC example](https://github.com/taosdata/TDengine/tree/develop/examples/JDBC) +请参考：[JDBC example](https://github.com/taosdata/TDengine/tree/3.0/examples/JDBC) ## 最近更新记录 From aace72a06c9146ab82b45d68b8a2bae5a3ae0866 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 9 Aug 2022 14:34:22 +0800 Subject: [PATCH 069/165] fix: modify udf document --- docs/zh/07-develop/09-udf.md | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 57c5618d54..08da9e296c 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -10,10 +10,12 @@ TDengine 支持通过 C/C++ 语言进行 UDF 定义。接下来结合示例讲 用户可以通过 UDF 实现两类函数： 标量函数和聚合函数。标量函数对每行数据返回一个值，如求绝对值 abs，正弦函数 sin，字符串拼接函数 concat 等。聚合函数对多行数据进行返回一个值，如求平均数 avg，最大值 max 等。 -实现udf时，需要实现规定的接口函数。接口函数的名称是 udf 名称，或者是 udf 名称和特定后缀（_start, _finish, _init, _destroy)的连接后。以下列表中的scalarfn，aggfn, udf需要替换成udf函数名。 -- 标量函数需要实现标量接口函数 scalarfn， +实现 UDF 时，需要实现规定的接口函数 +- 标量函数需要实现标量接口函数 scalarfn 。 - 聚合函数需要实现聚合接口函数 aggfn_start ， aggfn ， aggfn_finish。 -- 无论标量函数还是聚合函数，如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destory。 +- 如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destory。 + +接口函数的名称是 UDF 名称，或者是 UDF 名称和特定后缀（_start, _finish, _init, _destroy)的连接。列表中的scalarfn，aggfn, udf需要替换成udf函数名。 ## 实现标量函数 标量函数实现模板如下 @@ -102,17 +104,19 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。 接口函数的名称是 udf 名称，或者是 udf 名称和特定后缀（_start, _finish, _init, _destroy)的连接。以下描述中函数名称中的 scalarfn，aggfn, udf 需要替换成udf函数名。 -接口函数返回值表示是否成功，如果错误返回错误代码，错误代码见taoserror.h。参数类型见数据结构定义。 +接口函数返回值表示是否成功，如果错误返回错误代码，错误代码见taoserror.h。 + +接口函数参数类型见数据结构定义。 ### 标量接口函数 `int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn)` - 其中 udf 是函数名的占位符，以上述模板实现的函数对行数据块进行标量计算。 + 其中 scalarFn 是函数名的占位符。这个函数对数据块进行标量计算，通过设置resultColumn结构体中的变量设置值 -- 其中各参数的具体含义是： +参数的具体含义是： - inputDataBlock: 输入的数据块 - - resultColumn: 输出列 + - resultColumn: 输出列。输出列 ### 聚合接口函数 @@ -121,22 +125,22 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。 `int32_t aggfn(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf)` `int32_t aggfn_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result)` -其中 aggfn 是函数名的占位符。其中各参数的具体含义是： +其中 aggfn 是函数名的占位符。首先调用aggfn_start生成结果buffer，然后相关的数据会被分为多个行数据块，对每个数据块调用 aggfn 用数据块更新中间结果，最后再调用 aggfn_finish 从中间结果产生最终结果，最终结果只能含 0 或 1 条结果数据。 + +参数的具体含义是： - interBuf：中间结果 buffer。 - inputBlock：输入的数据块。 - newInterBuf：新的中间结果buffer。 - result：最终结果。 -其计算过程为：首先调用aggfn_start生成结果buffer，然后相关的数据会被分为多个行数据块，对每个行数据块调用 aggfn 用数据块更新中间结果，最后再调用 aggfn_finish 从中间结果产生最终结果，最终结果只能含 0 或 1 条结果数据。 - ### UDF 初始化和销毁 `int32_t udf_init()` `int32_t udf_destroy()` -其中 udf 是函数名的占位符，可以替换成自己的函数名。udf_init 完成初始化工作。 udf_destroy 完成清理工作。如果没有初始化工作，无需定义udf_init函数。如果没有清理工作，无需定义udf_destroy函数。 +其中 udf 是函数名的占位符。udf_init 完成初始化工作。 udf_destroy 完成清理工作。如果没有初始化工作，无需定义udf_init函数。如果没有清理工作，无需定义udf_destroy函数。 ## UDF 数据结构 @@ -190,10 +194,10 @@ typedef struct SUdfInterBuf { 数据结构说明如下： - SUdfDataBlock 数据块包含行数 numOfRows 和列数 numCols。udfCols[i] (0 <= i <= numCols-1)表示每一列数据，类型为SUdfColumn*。 -- SUdfColumn 包含列的数据类型定义 colMeta 和列的数据colData。 +- SUdfColumn 包含列的数据类型定义 colMeta 和列的数据 colData。 - SUdfColumnMeta 成员定义同 taos.h 数据类型定义。 -- SUdfColumnData 数据可以变长，varLenCol定义变长数据，fixLenCol定义定长数据。 -- SUdfInterBuf 定义中间结构buffer，以及buffer中结果个数 numOfResult +- SUdfColumnData 数据可以变长，varLenCol 定义变长数据，fixLenCol 定义定长数据。 +- SUdfInterBuf 定义中间结构 buffer，以及 buffer 中结果个数 numOfResult 为了更好的操作以上数据结构，提供了一些便利函数，定义在 taosudf.h。 From 9b1201a61ae00d5ca919d11dbda4aabf74b36258 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 9 Aug 2022 15:18:21 +0800 Subject: [PATCH 070/165] fix: slimit error --- source/libs/planner/src/planLogicCreater.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 12646c773c..875ec21b24 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -469,7 +469,8 @@ static int32_t createGroupKeysFromPartKeys(SNodeList* pPartKeys, SNodeList** pOu } static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { - return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; + return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP + : GROUP_ACTION_NONE; } static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { From dbf04b46dfd23a3c45b56f635daad46c3881fa5c Mon Sep 17 00:00:00 2001 From: Yang Zhao Date: Tue, 9 Aug 2022 15:34:50 +0800 Subject: [PATCH 071/165] docs: udpate python connector doc for 3.0 --- docs/zh/14-reference/03-connector/python.mdx | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/docs/zh/14-reference/03-connector/python.mdx b/docs/zh/14-reference/03-connector/python.mdx index a77dc71db7..d7b17dc74a 100644 --- a/docs/zh/14-reference/03-connector/python.mdx +++ b/docs/zh/14-reference/03-connector/python.mdx @@ -306,8 +306,7 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 | [bind_row.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-row.py) | 参数绑定，一次绑定一行 | | [insert_lines.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/insert-lines.py) | InfluxDB 行协议写入 | | [json_tag.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/json-tag.py) | 使用 JSON 类型的标签 | -| [subscribe-async.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/subscribe-async.py) | 异步订阅 | -| [subscribe-sync.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/subscribe-sync.py) | 同步订阅 | +| [tmq.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq.py) | tmq 订阅 | ## 其它说明 @@ -326,23 +325,15 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 1. https://stackoverflow.com/questions/10611328/parsing-datetime-strings-containing-nanoseconds 2. https://www.python.org/dev/peps/pep-0564/ - -## 常见问题 - -欢迎[提问或报告问题](https://github.com/taosdata/taos-connector-python/issues)。 - ## 重要更新 -| 连接器版本 | 重要更新 | 发布日期 | -| ---------- | --------------------------------------------------------------------------------- | ---------- | -| 2.3.1 | 1. support TDengine REST API
2. remove support for Python version below 3.6 | 2022-04-28 | -| 2.2.5 | support timezone option when connect | 2022-04-13 | -| 2.2.2 | support sqlalchemy dialect plugin | 2022-03-28 | - - [**Release Notes**](https://github.com/taosdata/taos-connector-python/releases) ## API 参考 - [taos](https://docs.taosdata.com/api/taospy/taos/) - [taosrest](https://docs.taosdata.com/api/taospy/taosrest) + +## 常见问题 + +欢迎[提问或报告问题](https://github.com/taosdata/taos-connector-python/issues)。 From 9e498a7859ea354e82cc13caa4035e85d3ce0003 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 9 Aug 2022 15:40:39 +0800 Subject: [PATCH 072/165] fix: set none for unbound cols --- source/libs/parser/src/parInsert.c | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 4756f65bf5..f2d0ede486 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1966,12 +1966,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in } // set the null value for the columns that do not assign values if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } tdSRowEnd(pBuilder); #ifdef TD_DEBUG_PRINT_ROW @@ -2050,12 +2045,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu // set the null value for the columns that do not assign values if (rowEnd && (spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } if (rowEnd) { tdSRowEnd(pBuilder); @@ -2432,12 +2422,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols // set the null value for the columns that do not assign values if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, - spd->cols[i].toffset); - } - } + pBuilder->hasNone = true; } tdSRowEnd(pBuilder); From e9f06d49c971cc6fdc98330cd9f5eba3485c6b61 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 9 Aug 2022 15:56:17 +0800 Subject: [PATCH 073/165] doc: fix broken links --- docs/zh/14-reference/03-connector/java.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index c45d670bc6..f679e5a272 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -630,7 +630,7 @@ public void setNString(int columnIndex, ArrayList list, int size) throws ### 无模式写入 -从 2.2.0.0 版本开始，TDengine 增加了对无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](/reference/schemaless/)。 +TDengine 支持对无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](../../reference/schemaless/)。 **注意**： @@ -914,7 +914,7 @@ Query OK, 1 row(s) in set (0.000141s) **解决方法**：重新安装 64 位 JDK。 -4. 其它问题请参考 [FAQ](/train-faq/faq) +4. 其它问题请参考 [FAQ](../../../train-faq/faq) ## API 参考 From a3668eb7ef2d55550357a3356322d22ea0787f25 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 9 Aug 2022 16:05:00 +0800 Subject: [PATCH 074/165] doc: refine java.mdx --- docs/zh/14-reference/03-connector/java.mdx | 50 ++++++++-------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index f679e5a272..8cbe77d29c 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -9,7 +9,7 @@ description: TDengine Java 连接器基于标准 JDBC API 实现, 并提供原 import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -`taos-jdbcdriver` 是 TDengine 的官方 Java 语言连接器，Java 开发人员可以通过它开发存取 TDengine 数据库的应用软件。`taos-jdbcdriver` 实现了 JDBC driver 标准的接口，并提供两种形式的连接器。一种是通过 TDengine 客户端驱动程序（taosc）原生连接 TDengine 实例，支持数据写入、查询、订阅、schemaless 接口和参数绑定接口等功能，一种是通过 taosAdapter 提供的 REST 接口连接 TDengine 实例（2.4.0.0 及更高版本）。REST 连接实现的功能集合和原生连接有少量不同。 +`taos-jdbcdriver` 是 TDengine 的官方 Java 语言连接器，Java 开发人员可以通过它开发存取 TDengine 数据库的应用软件。`taos-jdbcdriver` 实现了 JDBC driver 标准的接口，并提供两种形式的连接器。一种是通过 TDengine 客户端驱动程序（taosc）原生连接 TDengine 实例，支持数据写入、查询、订阅、schemaless 接口和参数绑定接口等功能，一种是通过 taosAdapter 提供的 REST 接口连接 TDengine 实例。REST 连接实现的功能集合和原生连接有少量不同。 ![TDengine Database Connector Java](tdengine-jdbc-connector.webp) @@ -41,19 +41,19 @@ REST 连接支持所有能运行 Java 的平台。 TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下： -| TDengine DataType | JDBCType （driver 版本 < 2.0.24） | JDBCType （driver 版本 >= 2.0.24） | -| ----------------- | --------------------------------- | ---------------------------------- | -| TIMESTAMP | java.lang.Long | java.sql.Timestamp | -| INT | java.lang.Integer | java.lang.Integer | -| BIGINT | java.lang.Long | java.lang.Long | -| FLOAT | java.lang.Float | java.lang.Float | -| DOUBLE | java.lang.Double | java.lang.Double | -| SMALLINT | java.lang.Short | java.lang.Short | -| TINYINT | java.lang.Byte | java.lang.Byte | -| BOOL | java.lang.Boolean | java.lang.Boolean | -| BINARY | java.lang.String | byte array | -| NCHAR | java.lang.String | java.lang.String | -| JSON | - | java.lang.String | +| TDengine DataType | JDBCType | +| ----------------- | ---------------------------------- | +| TIMESTAMP | java.sql.Timestamp | +| INT | java.lang.Integer | +| BIGINT | java.lang.Long | +| FLOAT | java.lang.Float | +| DOUBLE | java.lang.Double | +| SMALLINT | java.lang.Short | +| TINYINT | java.lang.Byte | +| BOOL | java.lang.Boolean | +| BINARY | byte array | +| NCHAR | java.lang.String | +| JSON | java.lang.String | **注意**：JSON 类型仅在 tag 中支持。 @@ -198,7 +198,7 @@ url 中的配置参数如下： - user：登录 TDengine 用户名，默认值 'root'。 - password：用户登录密码，默认值 'taosdata'。 -- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。从 taos-jdbcdriver-2.0.38 和 TDengine 2.4.0.12 版本开始，JDBC REST 连接增加批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 +- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。从 taos-jdbcdriver-2.0.38 开始，JDBC REST 连接增加批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 - httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 @@ -216,7 +216,7 @@ url 中的配置参数如下： INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFrancisco') VALUES(now, 24.6); ``` -- 从 taos-jdbcdriver-2.0.36 和 TDengine 2.2.0.0 版本开始，如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为 jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature) tags('California.SanFrancisco') values(now, 24.6); +- 从 taos-jdbcdriver-2.0.36 开始，如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为 jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature) tags('California.SanFrancisco') values(now, 24.6); ::: @@ -362,7 +362,7 @@ JDBC 连接器可能报错的错误码包括 3 种：JDBC driver 本身的报错 ### 通过参数绑定写入数据 -从 2.1.2.0 版本开始，TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据写入（INSERT）场景的支持。采用这种方式写入数据时，能避免 SQL 语法解析的资源消耗，从而在很多情况下显著提升写入性能。 +TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据写入（INSERT）场景的支持。采用这种方式写入数据时，能避免 SQL 语法解析的资源消耗，从而在很多情况下显著提升写入性能。 **注意**： @@ -630,7 +630,7 @@ public void setNString(int columnIndex, ArrayList list, int size) throws ### 无模式写入 -TDengine 支持对无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](../../reference/schemaless/)。 +TDengine 支持无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](../reference/schemaless/)。 **注意**： @@ -859,20 +859,6 @@ public static void main(String[] args) throws Exception { > 更多 druid 使用问题请查看[官方说明](https://github.com/alibaba/druid)。 -**注意事项：** - -- TDengine `v1.6.4.1` 版本开始提供了一个专门用于心跳检测的函数 `select server_status()`，所以在使用连接池时推荐使用 `select server_status()` 进行 Validation Query。 - -如下所示，`select server_status()` 执行成功会返回 `1`。 - -```sql -taos> select server_status(); -server_status()| -================ -1 | -Query OK, 1 row(s) in set (0.000141s) -``` - ### 更多示例程序 示例程序源码位于 `TDengine/examples/JDBC` 下: From 1d90aff252f38679b087ad0bf25c6e8028a499f5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 9 Aug 2022 16:13:51 +0800 Subject: [PATCH 075/165] fix(query): fix interp linear interpolation after range issue --- source/libs/executor/src/timewindowoperator.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index df5cc3324f..51fea5ac24 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2395,10 +2395,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); break; } - } else { - // ignore current row, and do nothing } - } else { // it is the last row of current block } } else { // non-linear interpolation pSliceInfo->current = @@ -2418,6 +2415,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { doKeepLinearInfo(pSliceInfo, pBlock, i); + // no need to increate pSliceInfo->current here //pSliceInfo->current = // taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (i < pBlock->info.rows - 1) { @@ -2436,10 +2434,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); break; } - } else { - // ignore current row, and do nothing } - } else { // it is the last row of current block } } else { // non-linear interpolation if (i < pBlock->info.rows - 1) { @@ -2518,10 +2513,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); break; } - } else { - // ignore current row, and do nothing } - } else { // it is the last row of current block } } else { // non-linear interpolation pSliceInfo->current = @@ -2541,7 +2533,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } // check if need to interpolate after ts range - // except for fill(next) + // except for fill(next), fill(linear) while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = From 52a620fbbafd03efe0087e6decdcb62c742be3a3 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Tue, 9 Aug 2022 16:18:10 +0800 Subject: [PATCH 076/165] Update java.mdx --- docs/zh/14-reference/03-connector/java.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index 8cbe77d29c..7a107bd04d 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -630,7 +630,7 @@ public void setNString(int columnIndex, ArrayList list, int size) throws ### 无模式写入 -TDengine 支持无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](../reference/schemaless/)。 +TDengine 支持无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](../../schemaless)。 **注意**： From 41d2c922311542eb0a393da065df3d0c52229f50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 16:54:05 +0800 Subject: [PATCH 077/165] fix(query): copy null value. --- source/common/src/tdatablock.c | 4 ---- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9a1d761e0d..dba30bb876 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1329,10 +1329,6 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); - if (pSrc->pData == NULL) { - continue; - } - colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 714af80e83..37861a0e0a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -182,6 +182,7 @@ static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes); + tsdbInfo("-------------------%d\n", pCol->info.bytes); } } @@ -674,6 +675,7 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ colDataAppendNULL(pColInfoData, rowIndex); } else { varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); + ASSERT(pColVal->value.nData <= pColInfoData->info.bytes); memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); colDataAppend(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); } From 41e2538ade8f83f42ddb7e22e86a17809f9c1a84 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 9 Aug 2022 16:54:59 +0800 Subject: [PATCH 078/165] scalable bloom filter encode & decode --- include/libs/stream/tstreamUpdate.h | 2 + include/util/tbloomfilter.h | 3 + include/util/tscalablebf.h | 3 +- source/libs/stream/src/streamUpdate.c | 108 ++++++++++++++++++ source/libs/stream/test/tstreamUpdateTest.cpp | 107 +++++++++++++++-- source/util/src/tbloomfilter.c | 37 +++++- source/util/src/tscalablebf.c | 42 ++++++- 7 files changed, 285 insertions(+), 17 deletions(-) diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index a4728e6382..78543118da 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -47,6 +47,8 @@ bool updateInfoIgnore(SUpdateInfo *pInfo, STimeWindow* pWin, uint64_t groupId, u void updateInfoDestroy(SUpdateInfo *pInfo); void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); +int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); +int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); #ifdef __cplusplus } diff --git a/include/util/tbloomfilter.h b/include/util/tbloomfilter.h index b168da594a..c9ca905f82 100644 --- a/include/util/tbloomfilter.h +++ b/include/util/tbloomfilter.h @@ -17,6 +17,7 @@ #define _TD_UTIL_BLOOMFILTER_H_ #include "os.h" +#include "tencode.h" #include "thash.h" #ifdef __cplusplus @@ -42,6 +43,8 @@ int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf, void tBloomFilterDestroy(SBloomFilter *pBF); void tBloomFilterDump(const SBloomFilter *pBF); bool tBloomFilterIsFull(const SBloomFilter *pBF); +int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder* pEncoder); +SBloomFilter* tBloomFilterDecode(SDecoder* pDecoder); #ifdef __cplusplus } diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index 8f88f65048..1386f840a8 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -33,7 +33,8 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); void tScalableBfDestroy(SScalableBf *pSBf); -void tScalableBfDump(const SScalableBf *pSBf); +int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder* pEncoder); +SScalableBf* tScalableBfDecode(SDecoder* pDecoder); #ifdef __cplusplus } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index ff1ef7b4b9..c686fa05ce 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -14,6 +14,7 @@ */ #include "tstreamUpdate.h" +#include "tencode.h" #include "ttime.h" #include "query.h" @@ -250,3 +251,110 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { tScalableBfDestroy(pInfo->pCloseWinSBF); pInfo->pCloseWinSBF = NULL; } + +int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) { + ASSERT(pInfo); + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + + int32_t size = taosArrayGetSize(pInfo->pTsBuckets); + if (tEncodeI32(&encoder, size) < 0) return -1; + for (int32_t i = 0; i < size; i++) { + TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->pTsBuckets, i); + if (tEncodeI64(&encoder, *pTs) < 0) return -1; + } + + if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) return -1; + + int32_t sBfSize = taosArrayGetSize(pInfo->pTsSBFs); + if (tEncodeI32(&encoder, sBfSize) < 0) return -1; + for (int32_t i = 0; i < sBfSize; i++) { + SScalableBf* pSBf = taosArrayGetP(pInfo->pTsSBFs, i); + if (tScalableBfEncode(pSBf, &encoder) < 0) return -1; + } + + if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->interval) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->watermark) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->minTS) < 0) return -1; + + if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) return -1; + + int32_t mapSize = taosHashGetSize(pInfo->pMap); + if (tEncodeI32(&encoder, mapSize) < 0) return -1; + void* pIte = NULL; + size_t keyLen = 0; + while ((pIte = taosHashIterate(pInfo->pMap, pIte)) != NULL) { + void* key = taosHashGetKey(pIte, &keyLen); + if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) return -1; + if (tEncodeI64(&encoder, *(TSKEY*)pIte) < 0) return -1; + } + + if (tEncodeI64(&encoder, pInfo->scanWindow.skey) < 0) return -1; + if (tEncodeI64(&encoder, pInfo->scanWindow.ekey) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->scanGroupId) < 0) return -1; + if (tEncodeU64(&encoder, pInfo->maxVersion) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { + ASSERT(pInfo); + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + if (tStartDecode(&decoder) < 0) return -1; + + int32_t size = 0; + if (tDecodeI32(&decoder, &size) < 0) return -1; + pInfo->pTsBuckets = taosArrayInit(size, sizeof(TSKEY)); + TSKEY ts = INT64_MIN; + for (int32_t i = 0; i < size; i++) { + if (tDecodeI64(&decoder, &ts) < 0) return -1; + taosArrayPush(pInfo->pTsBuckets, &ts); + } + + if (tDecodeU64(&decoder, &pInfo->numBuckets) < 0) return -1; + + int32_t sBfSize = 0; + if (tDecodeI32(&decoder, &sBfSize) < 0) return -1; + pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void *)); + for (int32_t i = 0; i < sBfSize; i++) { + SScalableBf* pSBf = tScalableBfDecode(&decoder); + if (!pSBf) return -1; + taosArrayPush(pInfo->pTsSBFs, &pSBf); + } + + if (tDecodeU64(&decoder, &pInfo->numSBFs) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->interval) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->watermark) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->minTS) < 0) return -1; + pInfo->pCloseWinSBF = tScalableBfDecode(&decoder); + + int32_t mapSize = 0; + if (tDecodeI32(&decoder, &mapSize) < 0) return -1; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pMap = taosHashInit(mapSize, hashFn, true, HASH_NO_LOCK); + uint64_t uid = 0; + ts = INT64_MIN; + for(int32_t i = 0; i < mapSize; i++) { + if (tDecodeU64(&decoder, &uid) < 0) return -1; + if (tDecodeI64(&decoder, &ts) < 0) return -1; + taosHashPut(pInfo->pMap, &uid, sizeof(uint64_t), &ts, sizeof(TSKEY)); + } + ASSERT(mapSize == taosHashGetSize(pInfo->pMap)); + + if (tDecodeI64(&decoder, &pInfo->scanWindow.skey) < 0) return -1; + if (tDecodeI64(&decoder, &pInfo->scanWindow.ekey) < 0) return -1; + if (tDecodeU64(&decoder, &pInfo->scanGroupId) < 0) return -1; + if (tDecodeU64(&decoder, &pInfo->maxVersion) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} diff --git a/source/libs/stream/test/tstreamUpdateTest.cpp b/source/libs/stream/test/tstreamUpdateTest.cpp index 93e114db02..933549b8a6 100644 --- a/source/libs/stream/test/tstreamUpdateTest.cpp +++ b/source/libs/stream/test/tstreamUpdateTest.cpp @@ -6,11 +6,37 @@ using namespace std; #define MAX_NUM_SCALABLE_BF 100000 +bool equalSBF(SScalableBf* left, SScalableBf* right) { + if (left->growth != right->growth) return false; + if (left->numBits != right->numBits) return false; + int lsize = taosArrayGetSize(left->bfArray); + int rsize = taosArrayGetSize(right->bfArray); + if (lsize != rsize) return false; + for (int32_t i = 0; i < lsize; i++) { + SBloomFilter* pLeftBF = (SBloomFilter*)taosArrayGetP(left->bfArray, i); + SBloomFilter* pRightBF = (SBloomFilter*)taosArrayGetP(right->bfArray, i); + if (pLeftBF->errorRate != pRightBF->errorRate) return false; + if (pLeftBF->expectedEntries != pRightBF->expectedEntries) return false; + if (pLeftBF->hashFn1 != pRightBF->hashFn1) return false; + if (pLeftBF->hashFn2 != pRightBF->hashFn2) return false; + if (pLeftBF->hashFunctions != pRightBF->hashFunctions) return false; + if (pLeftBF->numBits != pRightBF->numBits) return false; + if (pLeftBF->numUnits != pRightBF->numUnits) return false; + if (pLeftBF->size != pRightBF->size) return false; + uint64_t* leftUint = (uint64_t*) pLeftBF->buffer; + uint64_t* rightUint = (uint64_t*) pRightBF->buffer; + for (int32_t j = 0; j < pLeftBF->numUnits; j++) { + if (leftUint[j] != rightUint[j]) return false; + } + } + return true; +} + TEST(TD_STREAM_UPDATE_TEST, update) { - int64_t interval = 20 * 1000; - int64_t watermark = 10 * 60 * 1000; + const int64_t interval = 20 * 1000; + const int64_t watermark = 10 * 60 * 1000; SUpdateInfo *pSU = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark); - GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,1, 0), true); + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,1, 0), false); GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,1, -1), true); for(int i=0; i < 1024; i++) { @@ -31,15 +57,16 @@ TEST(TD_STREAM_UPDATE_TEST, update) { GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,i, 1), true); } + TSKEY uid = 0; for(int i=3; i < 1024; i++) { - GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,0, i), false); + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU, uid, i), false); } - GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU->pTsBuckets,0), 1023); + GTEST_ASSERT_EQ(*(TSKEY*)taosHashGet(pSU->pMap, &uid, sizeof(uint64_t)), 1023); for(int i=3; i < 1024; i++) { - GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU,0, i), true); + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU, uid, i), true); } - GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU->pTsBuckets,0), 1023); + GTEST_ASSERT_EQ(*(TSKEY*)taosHashGet(pSU->pMap, &uid, sizeof(uint64_t)), 1023); SUpdateInfo *pSU1 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark); for(int i=1; i <= watermark / interval; i++) { @@ -75,7 +102,8 @@ TEST(TD_STREAM_UPDATE_TEST, update) { GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU2, 1, i * interval + 5), false); GTEST_ASSERT_EQ(pSU2->minTS, (i-(pSU2->numSBFs-1))*interval); GTEST_ASSERT_EQ(pSU2->numSBFs, watermark / interval); - GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU2->pTsBuckets,1), i * interval + 5); + TSKEY uid2 = 1; + GTEST_ASSERT_EQ(*(TSKEY*)taosHashGet(pSU2->pMap, &uid2, sizeof(uint64_t)), i * interval + 5); } SUpdateInfo *pSU3 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark); @@ -84,7 +112,8 @@ TEST(TD_STREAM_UPDATE_TEST, update) { GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU3, i, i * interval + 5 * j), false); GTEST_ASSERT_EQ(pSU3->minTS, 0); GTEST_ASSERT_EQ(pSU3->numSBFs, watermark / interval); - GTEST_ASSERT_EQ(*(int64_t*)taosArrayGet(pSU3->pTsBuckets, i), i * interval + 5 * j); + uint64_t uid3 = i; + GTEST_ASSERT_EQ(*(TSKEY*)taosHashGet(pSU3->pMap, &uid3, sizeof(uint64_t)), i * interval + 5 * j); SScalableBf *pSBF = (SScalableBf *)taosArrayGetP(pSU3->pTsSBFs, i); SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF->bfArray, 0); GTEST_ASSERT_EQ(pBF->size, j); @@ -92,13 +121,66 @@ TEST(TD_STREAM_UPDATE_TEST, update) { } SUpdateInfo *pSU4 = updateInfoInit(-1, TSDB_TIME_PRECISION_MILLI, -1); - GTEST_ASSERT_EQ(pSU4->watermark, MAX_NUM_SCALABLE_BF * pSU4->interval); + GTEST_ASSERT_EQ(pSU4->watermark, pSU4->interval); GTEST_ASSERT_EQ(pSU4->interval, MILLISECOND_PER_MINUTE); SUpdateInfo *pSU5 = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); - GTEST_ASSERT_EQ(pSU5->watermark, MAX_NUM_SCALABLE_BF * pSU4->interval); + GTEST_ASSERT_EQ(pSU5->watermark, pSU4->interval); GTEST_ASSERT_EQ(pSU5->interval, MILLISECOND_PER_MINUTE); + SUpdateInfo *pSU7 = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark); + updateInfoAddCloseWindowSBF(pSU7); + for(int64_t i = 1; i < 2048000; i++) { + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU7,i, i), false); + } + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU7,100, 1), true); + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU7,110, 10), true); + GTEST_ASSERT_EQ(updateInfoIsUpdated(pSU7,200, 20), true); + + int32_t bufLen = updateInfoSerialize(NULL, 0, pSU7); + void* buf = taosMemoryCalloc(1, bufLen); + int32_t resSize = updateInfoSerialize(buf, bufLen, pSU7); + + SUpdateInfo *pSU6 = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); + int32_t desSize = updateInfoDeserialize(buf, bufLen, pSU6); + GTEST_ASSERT_EQ(desSize, 0); + + GTEST_ASSERT_EQ(pSU7->interval, pSU6->interval); + GTEST_ASSERT_EQ(pSU7->maxVersion, pSU6->maxVersion); + GTEST_ASSERT_EQ(pSU7->minTS, pSU6->minTS); + GTEST_ASSERT_EQ(pSU7->numBuckets, pSU6->numBuckets); + GTEST_ASSERT_EQ(pSU7->numSBFs, pSU6->numSBFs); + GTEST_ASSERT_EQ(pSU7->scanGroupId, pSU6->scanGroupId); + GTEST_ASSERT_EQ(pSU7->scanWindow.ekey, pSU6->scanWindow.ekey); + GTEST_ASSERT_EQ(pSU7->scanWindow.skey, pSU6->scanWindow.skey); + GTEST_ASSERT_EQ(pSU7->watermark, pSU6->watermark); + GTEST_ASSERT_EQ(equalSBF(pSU7->pCloseWinSBF, pSU6->pCloseWinSBF), true); + + int32_t mapSize = taosHashGetSize(pSU7->pMap); + GTEST_ASSERT_EQ(mapSize, taosHashGetSize(pSU6->pMap)); + void* pIte = NULL; + size_t keyLen = 0; + while ((pIte = taosHashIterate(pSU7->pMap, pIte)) != NULL) { + void* key = taosHashGetKey(pIte, &keyLen); + void* value6 = taosHashGet(pSU6->pMap, key, keyLen); + GTEST_ASSERT_EQ(*(TSKEY*)pIte, *(TSKEY*)value6); + } + + int32_t buSize = taosArrayGetSize(pSU7->pTsBuckets); + GTEST_ASSERT_EQ(buSize, taosArrayGetSize(pSU6->pTsBuckets)); + for (int32_t i = 0; i < buSize; i++) { + TSKEY ts1 = *(TSKEY*)taosArrayGet(pSU7->pTsBuckets, i); + TSKEY ts2 = *(TSKEY*)taosArrayGet(pSU6->pTsBuckets, i); + GTEST_ASSERT_EQ(ts1, ts2); + } + int32_t lSize = taosArrayGetSize(pSU7->pTsSBFs); + int32_t rSize = taosArrayGetSize(pSU6->pTsSBFs); + GTEST_ASSERT_EQ(lSize, rSize); + for (int32_t i = 0; i < lSize; i++) { + SScalableBf* pLeftSBF = (SScalableBf*)taosArrayGetP(pSU7->pTsSBFs, i); + SScalableBf* pRightSBF = (SScalableBf*)taosArrayGetP(pSU6->pTsSBFs, i); + GTEST_ASSERT_EQ(equalSBF(pLeftSBF, pRightSBF), true); + } updateInfoDestroy(pSU); updateInfoDestroy(pSU1); @@ -106,6 +188,9 @@ TEST(TD_STREAM_UPDATE_TEST, update) { updateInfoDestroy(pSU3); updateInfoDestroy(pSU4); updateInfoDestroy(pSU5); + updateInfoDestroy(pSU6); + updateInfoDestroy(pSU7); + } int main(int argc, char* argv[]) { diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 52c541ae2e..945cb58fcc 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -108,8 +108,41 @@ void tBloomFilterDestroy(SBloomFilter *pBF) { taosMemoryFree(pBF); } -void tBloomFilterDump(const struct SBloomFilter *pBF) { -// ToDo +int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder* pEncoder) { + if (tEncodeU32(pEncoder, pBF->hashFunctions) < 0) return -1; + if (tEncodeU64(pEncoder, pBF->expectedEntries) < 0) return -1; + if (tEncodeU64(pEncoder, pBF->numUnits) < 0) return -1; + if (tEncodeU64(pEncoder, pBF->numBits) < 0) return -1; + if (tEncodeU64(pEncoder, pBF->size) < 0) return -1; + for (uint64_t i = 0; i < pBF->numUnits; i++) { + uint64_t* pUnits = (uint64_t*)pBF->buffer; + if (tEncodeU64(pEncoder, pUnits[i]) < 0) return -1; + } + if (tEncodeDouble(pEncoder, pBF->errorRate) < 0) return -1; + return 0; +} + +SBloomFilter* tBloomFilterDecode(SDecoder* pDecoder) { + SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter)); + pBF->buffer = NULL; + if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) goto _error; + if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) goto _error; + if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) goto _error; + if (tDecodeU64(pDecoder, &pBF->numBits) < 0) goto _error; + if (tDecodeU64(pDecoder, &pBF->size) < 0) goto _error; + pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t)); + for (int32_t i = 0; i < pBF->numUnits; i++) { + uint64_t* pUnits = (uint64_t*)pBF->buffer; + if (tDecodeU64(pDecoder, pUnits + i) < 0) goto _error; + } + if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) goto _error; + pBF->hashFn1 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP); + pBF->hashFn2 = taosGetDefaultHashFunction(TSDB_DATA_TYPE_NCHAR); + return pBF; + +_error: + tBloomFilterDestroy(pBF); + return NULL; } bool tBloomFilterIsFull(const SBloomFilter *pBF) { diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 9ddac44e20..108eb34803 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -101,6 +101,42 @@ void tScalableBfDestroy(SScalableBf *pSBf) { taosMemoryFree(pSBf); } -void tScalableBfDump(const SScalableBf *pSBf) { - // Todo; -} \ No newline at end of file +int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder* pEncoder) { + if (!pSBf) { + if (tEncodeI32(pEncoder, 0) < 0) return -1; + return 0; + } + int32_t size = taosArrayGetSize(pSBf->bfArray); + if (tEncodeI32(pEncoder, size) < 0) return -1; + for (int32_t i = 0; i < size; i++) { + SBloomFilter* pBF = taosArrayGetP(pSBf->bfArray, i); + if (tBloomFilterEncode(pBF, pEncoder) < 0) return -1; + } + if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1; + if (tEncodeU64(pEncoder, pSBf->numBits) < 0) return -1; + return 0; +} + +SScalableBf* tScalableBfDecode(SDecoder* pDecoder) { + SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf)); + pSBf->bfArray = NULL; + int32_t size = 0; + if (tDecodeI32(pDecoder, &size) < 0) goto _error; + if (size == 0) { + tScalableBfDestroy(pSBf); + return NULL; + } + pSBf->bfArray = taosArrayInit(size * 2, sizeof(void *)); + for (int32_t i = 0; i < size; i++) { + SBloomFilter* pBF = tBloomFilterDecode(pDecoder); + if (!pBF) goto _error; + taosArrayPush(pSBf->bfArray, &pBF); + } + if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error; + if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error; + return pSBf; + +_error: + tScalableBfDestroy(pSBf); + return NULL; +} From 30b51e4041f696c512af0610c4f0000c56b72864 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 9 Aug 2022 16:57:41 +0800 Subject: [PATCH 079/165] enh: row optimization --- include/common/tdataformat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 311eb72b4d..a304dd93b2 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -200,8 +200,8 @@ struct STag { #if 1 //================================================================================================================================================ // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP -#define TD_SUPPORT_READ2 -#define TD_SUPPORT_BACK2 // suppport back compatibility of 2.0 +#undef TD_SUPPORT_READ2 +#undef TD_SUPPORT_BACK2 // suppport back compatibility of 2.0 #define TASSERT(x) ASSERT(x) From 8fb2948136146af1a8633c684cae5daae13443e6 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 9 Aug 2022 17:06:49 +0800 Subject: [PATCH 080/165] build: add taosBenchmark to deb --- README-CN.md | 11 ++--------- README.md | 11 ++--------- packaging/deb/makedeb.sh | 2 +- packaging/rpm/tdengine.spec | 2 +- packaging/tools/post.sh | 1 + 5 files changed, 7 insertions(+), 20 deletions(-) diff --git a/README-CN.md b/README-CN.md index ad31266bd4..b2ae777837 100644 --- a/README-CN.md +++ b/README-CN.md @@ -50,19 +50,12 @@ TDengine 目前 2.0 版服务器仅能在 Linux 系统上安装和运行，后 ## 安装工具 -### Ubuntu 16.04 及以上版本 & Debian： +### Ubuntu 18.04 及以上版本 & Debian： ```bash sudo apt-get install -y gcc cmake build-essential git libssl-dev ``` -### Ubuntu 14.04： - -```bash -sudo apt-get install -y gcc cmake3 build-essential git binutils-2.26 -export PATH=/usr/lib/binutils-2.26/bin:$PATH -``` - 编译或打包 JDBC 驱动源码，需安装 Java JDK 8 或以上版本和 Apache Maven 2.7 或以上版本。 安装 OpenJDK 8： @@ -89,7 +82,7 @@ taosTools 是用于 TDengine 的辅助工具软件集合。目前它包含 taosB sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config ``` -### CentOS 7： +### CentOS 7.9： ```bash sudo yum install -y gcc gcc-c++ make cmake git openssl-devel diff --git a/README.md b/README.md index 309bf16d77..2f38120b34 100644 --- a/README.md +++ b/README.md @@ -53,19 +53,12 @@ To build TDengine, use [CMake](https://cmake.org/) 3.0.2 or higher versions in t ## Install build dependencies -### Ubuntu 16.04 and above or Debian +### Ubuntu 18.04 and above or Debian ```bash sudo apt-get install -y gcc cmake build-essential git libssl-dev ``` -### Ubuntu 14.04 - -```bash -sudo apt-get install -y gcc cmake3 build-essential git binutils-2.26 -export PATH=/usr/lib/binutils-2.26/bin:$PATH -``` - To compile and package the JDBC driver source code, you should have a Java jdk-8 or higher and Apache Maven 2.7 or higher installed. To install openjdk-8: @@ -91,7 +84,7 @@ To build the [taosTools](https://github.com/taosdata/taos-tools) on Ubuntu/Debia sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config ``` -### CentOS 7 +### CentOS 7.9 ```bash sudo yum install epel-release diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 3e46a1aa55..6de475a4c0 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -60,7 +60,7 @@ cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin -#cp ${compile_dir}/build/bin/taosBenchmark ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/build/bin/taosBenchmark ${pkg_dir}${install_home_path}/bin if [ -f "${compile_dir}/build/bin/taosadapter" ]; then cp ${compile_dir}/build/bin/taosadapter ${pkg_dir}${install_home_path}/bin ||: diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 79e4b2edb8..d3d4bab0e6 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -69,7 +69,7 @@ cp %{_compiledir}/../packaging/tools/set_core.sh %{buildroot}%{homepath}/bin cp %{_compiledir}/../packaging/tools/taosd-dump-cfg.gdb %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin -#cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin if [ -f %{_compiledir}/build/bin/taosadapter ]; then cp %{_compiledir}/build/bin/taosadapter %{buildroot}%{homepath}/bin ||: diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index d637ab8d5a..aa80cfb86c 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -132,6 +132,7 @@ function install_bin() { [ -x ${bin_dir}/taosd ] && ${csudo}ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || : [ -x ${bin_dir}/taosadapter ] && ${csudo}ln -s ${bin_dir}/taosadapter ${bin_link_dir}/taosadapter || : [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosdemo || : + [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosBenchmark || : [ -x ${bin_dir}/TDinsight.sh ] && ${csudo}ln -sf ${bin_dir}/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${bin_dir}/taosdump ] && ${csudo}ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : [ -x ${bin_dir}/set_core.sh ] && ${csudo}ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : From 8b86d7342d89be79b32dd70a076e14cdad6fbdd5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 17:21:31 +0800 Subject: [PATCH 081/165] fix(query): check the sma load status, and return if succeed. --- source/libs/executor/src/scanoperator.c | 10 +++++++--- tests/script/tsim/testsuit.sim | 14 +++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b2fb80b68f..e401a1bd96 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -312,9 +312,13 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; - loadSMA = true; // mark the operator of load sma; + loadSMA = true; // mark the operation of load sma; bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); - if (!success) { // failed to load the block sma data, data block statistics does not exist, load data block instead + if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead + qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + return TSDB_CODE_SUCCESS; + } else { *status = FUNC_DATA_REQUIRED_DATA_LOAD; } } @@ -339,7 +343,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } } - // try to filter datablock according to current results + // try to filter data block according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index e0c46ae5fe..89a4babb0b 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -1,10 +1,10 @@ -run tsim/user/password.sim -run tsim/user/privilege_db.sim -run tsim/user/privilege_sysinfo.sim -run tsim/user/basic.sim -run tsim/table/basic1.sim -run tsim/trans/lossdata1.sim -run tsim/trans/create_db.sim +#run tsim/user/password.sim +#run tsim/user/privilege_db.sim +#run tsim/user/privilege_sysinfo.sim +#run tsim/user/basic.sim +#run tsim/table/basic1.sim +#run tsim/trans/lossdata1.sim +#run tsim/trans/create_db.sim run tsim/stable/alter_metrics.sim run tsim/stable/tag_modify.sim run tsim/stable/alter_comment.sim From bddb3e356d06def42f7430caeef971007f425d80 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 17:55:00 +0800 Subject: [PATCH 082/165] fix(query): expand the sink buffer. --- source/libs/executor/src/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 328a65bec4..497681ded1 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -353,7 +353,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, goto _error; } - SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100}; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 10000, .maxDataBlockNumPerQuery = 5000}; code = dsDataSinkMgtInit(&cfg); if (code != TSDB_CODE_SUCCESS) { goto _error; From 2119997661b8580949b59489fe9c7fd978fe79f2 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 9 Aug 2022 18:17:22 +0800 Subject: [PATCH 083/165] build: add taosBenchmark to deb --- cmake/cmake.platform | 4 ++-- examples/rust | 1 + tools/taos-tools | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) create mode 160000 examples/rust create mode 160000 tools/taos-tools diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 49e730a885..cd1be78fbc 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -97,13 +97,13 @@ IF ("${CPUTYPE}" STREQUAL "") ELSE () # if generate ARM version: # cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64 - IF (${CPUTYPE} MATCHES "aarch32") + IF (${CPUTYPE} MATCHES "aarch32" or ${CPUTYPE} MATCHES "arm32") SET(PLATFORM_ARCH_STR "arm") MESSAGE(STATUS "input cpuType: aarch32") ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_32") SET(TD_ARM_32 TRUE) - ELSEIF (${CPUTYPE} MATCHES "aarch64") + ELSEIF (${CPUTYPE} MATCHES "aarch64" or ${CPUTYPE} MATCHES "arm64") SET(PLATFORM_ARCH_STR "arm64") MESSAGE(STATUS "input cpuType: aarch64") ADD_DEFINITIONS("-D_TD_ARM_") diff --git a/examples/rust b/examples/rust new file mode 160000 index 0000000000..7ed7a97715 --- /dev/null +++ b/examples/rust @@ -0,0 +1 @@ +Subproject commit 7ed7a97715388fa144718764d6bf20f9bfc29a12 diff --git a/tools/taos-tools b/tools/taos-tools new file mode 160000 index 0000000000..3c7dafeea3 --- /dev/null +++ b/tools/taos-tools @@ -0,0 +1 @@ +Subproject commit 3c7dafeea3e558968165b73bee0f51024898e3da From 9c3e08ad8b3537f36a677f82d499402946e30eb0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 9 Aug 2022 18:27:04 +0800 Subject: [PATCH 084/165] fix tag filter --- source/libs/index/src/indexFilter.c | 126 +++++++++++++++++++++------- 1 file changed, 98 insertions(+), 28 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 96a42928d4..d9d303c1f3 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -358,9 +358,101 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { } return NULL; } -static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { - int ret = 0; +typedef union { + uint8_t u8; + uint16_t u16; + uint32_t u32; + uint64_t u64; + int8_t i8; + int16_t i16; + int32_t i32; + int64_t i64; + + double d; + float f; +} SDataTypeBuf; + +#define SIF_DATA_CONVERT(type, val, dst) \ + do { \ + if (type == TSDB_DATA_TYPE_DOUBLE) \ + dst = GET_DOUBLE_VAL(val); \ + else if (type == TSDB_DATA_TYPE_BIGINT) \ + dst = *(int64_t *)val; \ + else if (type == TSDB_DATA_TYPE_INT) \ + dst = *(int32_t *)val; \ + else if (type == TSDB_DATA_TYPE_SMALLINT) \ + dst = *(int16_t *)val; \ + else if (type == TSDB_DATA_TYPE_TINYINT) \ + dst = *(int8_t *)val; \ + else if (type == TSDB_DATA_TYPE_UTINYINT) \ + dst = *(uint8_t *)val; \ + else if (type == TSDB_DATA_TYPE_USMALLINT) \ + dst = *(uint16_t *)val; \ + else if (type == TSDB_DATA_TYPE_UINT) \ + dst = *(uint32_t *)val; \ + else if (type == TSDB_DATA_TYPE_UBIGINT) \ + dst = *(uint64_t *)val; \ + } while (0); + +static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { + int8_t ltype = left->colValType, rtype = right->colValType; + if (ltype == TSDB_DATA_TYPE_FLOAT) { + float f; + SIF_DATA_CONVERT(rtype, right->condValue, f); + typedata->f = f; + param->val = &typedata->f; + } else if (ltype == TSDB_DATA_TYPE_DOUBLE) { + double d; + SIF_DATA_CONVERT(rtype, right->condValue, d); + typedata->d = d; + param->val = &typedata->d; + } else if (ltype == TSDB_DATA_TYPE_BIGINT) { + int64_t i64; + SIF_DATA_CONVERT(rtype, right->condValue, i64); + typedata->i64 = i64; + param->val = &typedata->i64; + } else if (ltype == TSDB_DATA_TYPE_INT) { + int32_t i32; + SIF_DATA_CONVERT(rtype, right->condValue, i32); + typedata->i32 = i32; + param->val = &typedata->i32; + } else if (ltype == TSDB_DATA_TYPE_SMALLINT) { + int16_t i16; + + SIF_DATA_CONVERT(rtype, right->condValue, i16); + typedata->i16 = i16; + param->val = &typedata->i16; + } else if (ltype == TSDB_DATA_TYPE_TINYINT) { + int8_t i8; + SIF_DATA_CONVERT(rtype, right->condValue, i8) + typedata->i8 = i8; + param->val = &typedata->i8; + } else if (ltype == TSDB_DATA_TYPE_UBIGINT) { + uint64_t u64; + typedata->u64 = u64; + SIF_DATA_CONVERT(rtype, right->condValue, u64); + param->val = &typedata->u64; + + } else if (ltype == TSDB_DATA_TYPE_UINT) { + uint32_t u32; + typedata->u32 = u32; + SIF_DATA_CONVERT(rtype, right->condValue, u32); + param->val = &typedata->u32; + } else if (ltype == TSDB_DATA_TYPE_USMALLINT) { + uint16_t u16; + SIF_DATA_CONVERT(rtype, right->condValue, u16); + typedata->u16 = u16; + param->val = &typedata->u16; + } else if (ltype == TSDB_DATA_TYPE_UTINYINT) { + uint8_t u8; + typedata->u8 = u8; + SIF_DATA_CONVERT(rtype, right->condValue, u8); + param->val = &typedata->u8; + } +} +static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { + int ret = 0; SIndexMetaArg *arg = &output->arg; EIndexQueryType qtype = 0; SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); @@ -385,9 +477,9 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP .reverse = reverse, .filterFunc = filterFunc}; - char buf[128] = {0}; - float f = 0.0; - double d = 0.0; + char buf[128] = {0}; + + SDataTypeBuf typedata; if (IS_VAR_DATA_TYPE(left->colValType)) { if (!IS_VAR_DATA_TYPE(right->colValType)) { NUM_TO_STRING(right->colValType, right->condValue, sizeof(buf) - 2, buf + VARSTR_HEADER_SIZE); @@ -395,29 +487,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP param.val = buf; } } else { - // int8_t i8; int16_t i16; int32_t i32, uint8_t u8; uint16_t u16; uint32_t u32; - // if (right->colValType == TSDB_DATA_TYPE_TINYINT) { - // } - // if (left->colValType == TSDB_DATA_TYPE_FLOAT) { - // if (right->colValType == TSDB_DATA_TYPE_DOUBLE) { - // f = GET_DOUBLE_VAL(right->condValue); - // param.val = &f; - // } else if (right->colValType == TSDB_DATA_TYPE_BIGINT) { - // f = *(int64_t *)(right->condValue); - // param.val = &f; - // } else { - // f = *(int32_t *)(right->condValue); - // param.val = &f; - // } - // } else if (left->colValType == TSDB_DATA_TYPE_DOUBLE) { - // if (right->colValType == TSDB_DATA_TYPE_DOUBLE) { - // d = GET_DOUBLE_VAL(right->condValue); - // param.val = &d; - // } else if (right->colValType == TSDB_DATA_TYPE_BIGINT) { - // d = *(int64_t *)(right->condValue); - // param.val = &d; - // } - // } + sifSetFltParam(left, right, &typedata, ¶m); } ret = metaFilterTableIds(arg->metaEx, ¶m, output->result); } From d680da33806a3b0e3fbaac48a3dd01cbcfd73721 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 18:35:10 +0800 Subject: [PATCH 085/165] fix(query): fix sma info in query procesing. --- source/libs/executor/src/executor.c | 4 ++++ source/libs/executor/src/scanoperator.c | 3 +++ tests/script/tsim/parser/limit1_tb.sim | 4 ++++ tests/script/tsim/parser/mixed_blocks.sim | 1 + 4 files changed, 12 insertions(+) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 497681ded1..1fc0f5ff51 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -348,6 +348,8 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, taosThreadOnce(&initPoolOnce, initRefPool); atexit(cleanupRefPool); + qDebug("start to create subplan task, TID:0x%"PRIx64 " QID:0x%"PRIx64, taskId, pSubplan->id.queryId); + int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -372,6 +374,8 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } } + qDebug("subplan task create completed, TID:0x%"PRIx64 " QID:0x%"PRIx64, taskId, pSubplan->id.queryId); + _error: // if failed to add ref for all tables in this query, abort current query return code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e401a1bd96..a0f2560e7b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -343,6 +343,9 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } } + // free the sma info, since it should not be involved in later computing process. + taosMemoryFreeClear(pBlock->pBlockAgg); + // try to filter data block according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { diff --git a/tests/script/tsim/parser/limit1_tb.sim b/tests/script/tsim/parser/limit1_tb.sim index 1ef6a62291..36c91fe4c7 100644 --- a/tests/script/tsim/parser/limit1_tb.sim +++ b/tests/script/tsim/parser/limit1_tb.sim @@ -263,6 +263,9 @@ sql select sum(c1), avg(c2), stddev(c3), max(c4), min(c5), count(c6), first(c7), if $rows != 0 then return -1 endi + +print select sum(c1), avg(c2), stddev(c3), max(c4), min(c5), count(c6), first(c7), last(c8), last(c9) from $tb where ts >= $ts0 and ts <= $tsu and c1>1 and c2<9 and c3>2 and c4<8 and c5>4 and c6<6 limit 1 offset 0 + sql select sum(c1), avg(c2), stddev(c3), max(c4), min(c5), count(c6), first(c7), last(c8), last(c9) from $tb where ts >= $ts0 and ts <= $tsu and c1>1 and c2<9 and c3>2 and c4<8 and c5>4 and c6<6 limit 1 offset 0 if $rows != 1 then return -1 @@ -270,6 +273,7 @@ endi $val = $rowNum / 10 $val = $val * 5 if $data00 != $val then + print $data00, expect $val return -1 endi if $data01 != 5.000000000 then diff --git a/tests/script/tsim/parser/mixed_blocks.sim b/tests/script/tsim/parser/mixed_blocks.sim index 76ac7c1c54..80cdfcac99 100644 --- a/tests/script/tsim/parser/mixed_blocks.sim +++ b/tests/script/tsim/parser/mixed_blocks.sim @@ -92,6 +92,7 @@ if $data03 != 319 then return -1 endi if $data04 != 318.000000000 then + print expect 318.000000000, actual: $data04 return -1 endi if $data05 != 1 then From 4174add647273a3aa9753c77d9c2c4c19342084c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 9 Aug 2022 18:42:04 +0800 Subject: [PATCH 086/165] chore: udpate libtaos ws submodule for3.0 (#15901) * chore: add libtaos-ws for 3.0 * chore: update taosws-rs * chore: add libtaosws to install/remove script * chore: update taosws-rs * chore: update taosws-rs * chore: update taos-tools, taosws-rs for 3.0 * fix: packaging/tools/make_install.sh for 3.0 * chore: update taos-tools * chore: fix release script for 3.0 * chore: update taosws-rs for 3.0 * chore: add taows-rs submodule for 3.0 * chore: update taosws-rs for 3.0 * fix: install script support taosws for 3.0 * fix: script error handle for 3.0 * chore: update taosws-rs for 3.0 fix segfault * chore: change container_build for websocket build * fix: install script for taosws * fix: . * chore: update taosws-rs for 3.0 * chore: update taosws-rs for 3.0 * chore: update tools/CMakeLists.txt to allow compile taosws-rw on any platform * chore: taosws 648cc62 for 3.0 * chore: update taosws 29424d5 for 3.0 * chore: update cmake/taosws_CMakeLists.txt.in with new repo/commit --- cmake/taosws_CMakeLists.txt.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index de6409a8c6..c6d42b686c 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -1,8 +1,8 @@ # taosws-rs ExternalProject_Add(taosws-rs - GIT_REPOSITORY https://github.com/taosdata/taosws-rs.git - GIT_TAG 29424d5 + GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git + GIT_TAG 97c4bac SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 7b70f54a6e3c30accbd4e3fe50d5500f1b812b4e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 18:47:23 +0800 Subject: [PATCH 087/165] test: add log. --- source/libs/executor/src/scanoperator.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a0f2560e7b..a211542de1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2665,11 +2665,17 @@ _error: int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idStr) { + + int64_t st = taosGetTimestampUs(); + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { return code; } + int64_t st1 = taosGetTimestampUs(); + qDebug("generate queried table list completed, elapsed time:%.2f ms %s", (st1-st)/1000.0, idStr); + if (taosArrayGetSize(pTableListInfo->pTableList) == 0) { qDebug("no table qualified for query, %s" PRIx64, idStr); return TSDB_CODE_SUCCESS; @@ -2681,6 +2687,9 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return code; } + int64_t st2 = taosGetTimestampUs(); + qDebug("generate group id map completed, elapsed time:%.2f ms %s", (st2-st1)/1000.0, idStr); + return TSDB_CODE_SUCCESS; } From b3b10bbcf69c99fa1648d6f0385bab5fea1b4386 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 9 Aug 2022 16:48:36 +0800 Subject: [PATCH 088/165] test: adjust valgrind case --- tests/script/jenkins/basic.txt | 8 +-- tests/script/tsim/parser/alter1.sim | 2 + .../script/tsim/parser/columnValue_bigint.sim | 2 + tests/script/tsim/parser/columnValue_bool.sim | 1 + .../script/tsim/parser/columnValue_double.sim | 2 + .../script/tsim/parser/columnValue_float.sim | 2 + tests/script/tsim/parser/columnValue_int.sim | 2 + .../tsim/parser/columnValue_smallint.sim | 2 + .../tsim/parser/columnValue_tinyint.sim | 2 + .../script/tsim/parser/columnValue_unsign.sim | 1 + tests/script/tsim/parser/create_db.sim | 4 +- tests/script/tsim/valgrind/basic3.sim | 6 +- tests/script/tsim/valgrind/basic4.sim | 56 +++++++++---------- tests/script/tsim/valgrind/checkError6.sim | 3 + 14 files changed, 53 insertions(+), 40 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0a859b2045..23676153b5 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -84,7 +84,7 @@ ./test.sh -f tsim/insert/update0.sim ./test.sh -f tsim/insert/update1_sort_merge.sim -# ---- parser +# ---- parser ---- ./test.sh -f tsim/parser/alter__for_community_version.sim ./test.sh -f tsim/parser/alter_column.sim ./test.sh -f tsim/parser/alter_stable.sim @@ -106,7 +106,7 @@ ./test.sh -f tsim/parser/commit.sim ./test.sh -f tsim/parser/condition.sim ./test.sh -f tsim/parser/constCol.sim -#./test.sh -f tsim/parser/create_db.sim +./test.sh -f tsim/parser/create_db.sim ./test.sh -f tsim/parser/create_mt.sim ./test.sh -f tsim/parser/create_tb_with_tag_name.sim ./test.sh -f tsim/parser/create_tb.sim @@ -129,7 +129,7 @@ ./test.sh -f tsim/parser/import.sim ./test.sh -f tsim/parser/insert_multiTbl.sim ./test.sh -f tsim/parser/insert_tb.sim -# TD-17038 ./test.sh -f tsim/parser/interp.sim +# TD-18293 ./test.sh -f tsim/parser/interp.sim ./test.sh -f tsim/parser/join_manyblocks.sim ./test.sh -f tsim/parser/join_multitables.sim ./test.sh -f tsim/parser/join_multivnode.sim @@ -318,7 +318,7 @@ ./test.sh -f tsim/valgrind/checkError7.sim ./test.sh -f tsim/valgrind/checkUdf.sim -# --- vnode +# --- vnode ---- # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim # unsupport ./test.sh -f tsim/vnode/replica3_repeat.sim # unsupport ./test.sh -f tsim/vnode/replica3_vgroup.sim diff --git a/tests/script/tsim/parser/alter1.sim b/tests/script/tsim/parser/alter1.sim index d917f4b61e..6771b35eae 100644 --- a/tests/script/tsim/parser/alter1.sim +++ b/tests/script/tsim/parser/alter1.sim @@ -129,3 +129,5 @@ endi #if $rows != 0 then # return -1 #endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_bigint.sim b/tests/script/tsim/parser/columnValue_bigint.sim index ae97835dff..a196c61684 100644 --- a/tests/script/tsim/parser/columnValue_bigint.sim +++ b/tests/script/tsim/parser/columnValue_bigint.sim @@ -425,3 +425,5 @@ sql insert into st_bigint_e25 using mt_bigint tags (033) values (now, 00062) #sql_error alter table st_bigint_e23 set tag tagname="abc" #sql_error alter table st_bigint_e24 set tag tagname=" " #sql_error alter table st_bigint_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_bool.sim b/tests/script/tsim/parser/columnValue_bool.sim index d20c4efdc0..ffdb095cee 100644 --- a/tests/script/tsim/parser/columnValue_bool.sim +++ b/tests/script/tsim/parser/columnValue_bool.sim @@ -634,3 +634,4 @@ sql alter table st_bool_i4 set tag tagname="abc" sql alter table st_bool_i5 set tag tagname=" " sql alter table st_bool_i6 set tag tagname='' +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_double.sim b/tests/script/tsim/parser/columnValue_double.sim index dae64735ea..d74dda36dc 100644 --- a/tests/script/tsim/parser/columnValue_double.sim +++ b/tests/script/tsim/parser/columnValue_double.sim @@ -528,3 +528,5 @@ sql_error alter table st_double_e22 set tag tagname=abc sql alter table st_double_e23 set tag tagname="abc" sql alter table st_double_e24 set tag tagname=" " sql alter table st_double_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_float.sim b/tests/script/tsim/parser/columnValue_float.sim index 9b0ca4b186..a55e1aea2d 100644 --- a/tests/script/tsim/parser/columnValue_float.sim +++ b/tests/script/tsim/parser/columnValue_float.sim @@ -558,3 +558,5 @@ sql_error alter table st_float_e22 set tag tagname=abc sql alter table st_float_e23 set tag tagname="abc" sql alter table st_float_e24 set tag tagname=" " sql alter table st_float_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_int.sim b/tests/script/tsim/parser/columnValue_int.sim index 48d95f5ecb..34ad3d93fd 100644 --- a/tests/script/tsim/parser/columnValue_int.sim +++ b/tests/script/tsim/parser/columnValue_int.sim @@ -423,3 +423,5 @@ sql_error alter table st_int_e22 set tag tagname=abc sql_error alter table st_int_e23 set tag tagname="abc" sql_error alter table st_int_e24 set tag tagname=" " sql alter table st_int_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_smallint.sim b/tests/script/tsim/parser/columnValue_smallint.sim index ced486ba0b..39608a8efe 100644 --- a/tests/script/tsim/parser/columnValue_smallint.sim +++ b/tests/script/tsim/parser/columnValue_smallint.sim @@ -426,3 +426,5 @@ sql_error alter table st_smallint_e22 set tag tagname=abc sql_error alter table st_smallint_e23 set tag tagname="abc" sql_error alter table st_smallint_e24 set tag tagname=" " sql alter table st_smallint_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_tinyint.sim b/tests/script/tsim/parser/columnValue_tinyint.sim index bc1fcd3445..fe2734c245 100644 --- a/tests/script/tsim/parser/columnValue_tinyint.sim +++ b/tests/script/tsim/parser/columnValue_tinyint.sim @@ -424,3 +424,5 @@ sql_error alter table st_tinyint_e22 set tag tagname=abc sql_error alter table st_tinyint_e23 set tag tagname="abc" sql_error alter table st_tinyint_e24 set tag tagname=" " sql alter table st_tinyint_e25 set tag tagname='' + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/columnValue_unsign.sim b/tests/script/tsim/parser/columnValue_unsign.sim index a72b1082f6..758814bc2b 100644 --- a/tests/script/tsim/parser/columnValue_unsign.sim +++ b/tests/script/tsim/parser/columnValue_unsign.sim @@ -130,3 +130,4 @@ if $rows != 1 then return -1 endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/create_db.sim b/tests/script/tsim/parser/create_db.sim index 8d0bc3fe5b..d50b53b2f7 100644 --- a/tests/script/tsim/parser/create_db.sim +++ b/tests/script/tsim/parser/create_db.sim @@ -237,8 +237,8 @@ if $rows != 3 then return -1 endi sql show databases -print wallevel $data13_testwal -if $data13_testwal != 1 then +print wallevel $data20_testwal +if $data20_testwal != 1 then return -1 endi sql drop database testwal diff --git a/tests/script/tsim/valgrind/basic3.sim b/tests/script/tsim/valgrind/basic3.sim index 0913691a11..3a51186d55 100644 --- a/tests/script/tsim/valgrind/basic3.sim +++ b/tests/script/tsim/valgrind/basic3.sim @@ -50,11 +50,11 @@ while $i < $tbNum endw print =============== step3: tb -sql select tbcol5 - tbcol3 from tb1 -print =============== step4: stb -sql select tbcol5 - tbcol3 from stb +sql select sum(tbcol) from stb partition by tbname interval(1s) slimit 1 soffset 1; +sql select sum(tbcol) from stb partition by tbname interval(1s) slimit 2 soffset 4 limit 10 offset 1; +_OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check $null= diff --git a/tests/script/tsim/valgrind/basic4.sim b/tests/script/tsim/valgrind/basic4.sim index 8be96f769b..98c4f8f2a8 100644 --- a/tests/script/tsim/valgrind/basic4.sim +++ b/tests/script/tsim/valgrind/basic4.sim @@ -22,41 +22,35 @@ if $data(1)[4] != ready then goto step1 endi -print =============== step2: create db -sql create database d1 vgroups 2 buffer 3 -sql show databases -sql use d1 -sql show vgroups +$tbPrefix = tb +$tbNum = 5 +$rowNum = 10 -print =============== step3: create show stable -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) -sql show stables -if $rows != 1 then - return -1 -endi +print =============== step2: prepare data +sql create database db vgroups 2 +sql use db +sql create table if not exists stb (ts timestamp, tbcol int, tbcol2 float, tbcol3 double, tbcol4 binary(30), tbcol5 binary(30)) tags (tgcol int unsigned) -print =============== step4: create show table -sql create table ct1 using stb tags(1000) -sql create table ct2 using stb tags(2000) -sql create table ct3 using stb tags(3000) -sql show tables -if $rows != 3 then - return -1 -endi +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + sql insert into $tb values ($ms , $x , $x , $x , "abcd1234=-+*" , "123456 0" ) + $x = $x + 1 + endw -print =============== step5: insert data (null / update) -sql insert into ct1 values(now+0s, 10, 2.0, 3.0) -sql insert into ct1 values(now+1s, 11, 2.1, NULL)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) -sql insert into ct2 values(now+0s, 10, 2.0, 3.0) -sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) -sql insert into ct3 values('2021-01-01 00:00:00.000', NULL, NULL, 3.0) -sql insert into ct3 values('2022-03-02 16:59:00.010', 3 , 4, 5), ('2022-03-02 16:59:00.010', 33 , 4, 5), ('2022-04-01 16:59:00.011', 4, 4, 5), ('2022-04-01 16:59:00.011', 6, 4, 5), ('2022-03-06 16:59:00.013', 8, 4, 5); -sql insert into ct3 values('2022-03-02 16:59:00.010', 103, 1, 2), ('2022-03-02 16:59:00.010', 303, 3, 4), ('2022-04-01 16:59:00.011', 40, 5, 6), ('2022-04-01 16:59:00.011', 60, 4, 5), ('2022-03-06 16:59:00.013', 80, 4, 5); - -print =============== step6: query data= - -sql select * from stb where t1 between 1000 and 2500 + $cc = $x * 60000 + $ms = 1601481600000 + $cc + sql insert into $tb values ($ms , NULL , NULL , NULL , NULL , NULL ) + $i = $i + 1 +endw +print =============== step3: tb +sql select distinct(tbname), tgcol from stb; _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index 6e456148bf..70f55370d2 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -111,6 +111,9 @@ sql select _wstart, count(*) from tb1 session(ts, 1m) sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) sql select tbcol5 - tbcol3 from stb +sql select spread( tbcol2 )/44, spread(tbcol2), 0.204545455 * 44 from stb; +sql select min(tbcol) * max(tbcol) /4, sum(tbcol2) * apercentile(tbcol2, 20), apercentile(tbcol2, 33) + 52/9 from stb; + print =============== step5: explain sql explain analyze select ts from stb where -2; sql explain analyze select ts from tb1; From d0416f312f48d309a5f9299bdacc6e7cccd0cf6d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 9 Aug 2022 19:07:07 +0800 Subject: [PATCH 089/165] fix: tdb use sys memory when snapshot writing --- source/dnode/vnode/src/inc/vnodeInt.h | 4 ++-- source/dnode/vnode/src/meta/metaCommit.c | 9 ++++++--- source/dnode/vnode/src/meta/metaSnapshot.c | 2 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 47f7d209b3..2b154fce04 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -97,7 +97,7 @@ typedef struct STbUidStore STbUidStore; int metaOpen(SVnode* pVnode, SMeta** ppMeta); int metaClose(SMeta* pMeta); -int metaBegin(SMeta* pMeta); +int metaBegin(SMeta* pMeta, int8_t fromSys); int metaCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); @@ -187,7 +187,7 @@ int32_t smaAsyncPreCommit(SSma* pSma); int32_t smaAsyncCommit(SSma* pSma); int32_t smaAsyncPostCommit(SSma* pSma); int32_t smaDoRetention(SSma* pSma, int64_t now); -int32_t smaProcessFetch(SSma *pSma, void* pMsg); +int32_t smaProcessFetch(SSma* pSma, void* pMsg); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index b4987aea2b..85ed40970c 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -19,9 +19,12 @@ static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { return vnodeBuf static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } // begin a meta txn -int metaBegin(SMeta *pMeta) { - tdbTxnOpen(&pMeta->txn, 0, metaMalloc, metaFree, pMeta->pVnode->inUse, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - +int metaBegin(SMeta *pMeta, int8_t fromSys) { + if (fromSys) { + tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } else { + tdbTxnOpen(&pMeta->txn, 0, metaMalloc, metaFree, pMeta->pVnode->inUse, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } if (tdbBegin(pMeta->pEnv, &pMeta->txn) < 0) { return -1; } diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index e01f0e7c01..973c381407 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -145,7 +145,7 @@ int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWr pWriter->sver = sver; pWriter->ever = ever; - metaBegin(pMeta); + metaBegin(pMeta, 1); *ppWriter = pWriter; return code; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 460d0b6fbb..c8dc07af0a 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -42,7 +42,7 @@ int vnodeBegin(SVnode *pVnode) { pVnode->state.commitID++; // begin meta - if (metaBegin(pVnode->pMeta) < 0) { + if (metaBegin(pVnode->pMeta, 0) < 0) { vError("vgId:%d, failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } From 2398c9e81b0df5caf6af92627d673883fb543ab7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 9 Aug 2022 19:38:48 +0800 Subject: [PATCH 090/165] fix tag filter --- source/libs/index/src/indexFilter.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index d9d303c1f3..21aeaba70b 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -430,14 +430,14 @@ static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typeda param->val = &typedata->i8; } else if (ltype == TSDB_DATA_TYPE_UBIGINT) { uint64_t u64; - typedata->u64 = u64; SIF_DATA_CONVERT(rtype, right->condValue, u64); + typedata->u64 = u64; param->val = &typedata->u64; } else if (ltype == TSDB_DATA_TYPE_UINT) { uint32_t u32; - typedata->u32 = u32; SIF_DATA_CONVERT(rtype, right->condValue, u32); + typedata->u32 = u32; param->val = &typedata->u32; } else if (ltype == TSDB_DATA_TYPE_USMALLINT) { uint16_t u16; @@ -446,8 +446,8 @@ static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typeda param->val = &typedata->u16; } else if (ltype == TSDB_DATA_TYPE_UTINYINT) { uint8_t u8; - typedata->u8 = u8; SIF_DATA_CONVERT(rtype, right->condValue, u8); + typedata->u8 = u8; param->val = &typedata->u8; } } @@ -480,6 +480,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP char buf[128] = {0}; SDataTypeBuf typedata; + memset(&typedata, 0, sizeof(typedata)); if (IS_VAR_DATA_TYPE(left->colValType)) { if (!IS_VAR_DATA_TYPE(right->colValType)) { NUM_TO_STRING(right->colValType, right->condValue, sizeof(buf) - 2, buf + VARSTR_HEADER_SIZE); From 42b5069586c207626498c4dfc4f5b2f87a903c96 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 9 Aug 2022 19:41:16 +0800 Subject: [PATCH 091/165] test: valgrind case --- tests/script/tsim/valgrind/checkError8.sim | 152 +++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 tests/script/tsim/valgrind/checkError8.sim diff --git a/tests/script/tsim/valgrind/checkError8.sim b/tests/script/tsim/valgrind/checkError8.sim new file mode 100644 index 0000000000..7ca01bc3d0 --- /dev/null +++ b/tests/script/tsim/valgrind/checkError8.sim @@ -0,0 +1,152 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start -v +sql connect + +sql drop database if exists cdb +sql create database if not exists cdb +sql use cdb +sql create table stb1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb1 using stb1 tags(1,'1',1.0) +sql create table tb2 using stb1 tags(2,'2',2.0) +sql create table tb3 using stb1 tags(3,'3',3.0) +sql create table tb4 using stb1 tags(4,'4',4.0) +sql create table tb5 using stb1 tags(5,'5',5.0) +sql create table tb6 using stb1 tags(6,'6',6.0) + +sql insert into tb1 values ('2021-05-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb1 values ('2021-05-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb1 values ('2021-05-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb1 values ('2021-05-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb1 values ('2021-05-05 18:19:04',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb1 values ('2021-05-05 18:19:05',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb1 values ('2021-05-05 18:19:06',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb1 values ('2021-05-05 18:19:07',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb2 values ('2021-05-05 18:19:08',21,21.0,21,21,21,21.0,true ,'21','21') +sql insert into tb2 values ('2021-05-05 18:19:09',22,22.0,22,22,22,22.0,true ,'22','22') +sql insert into tb2 values ('2021-05-05 18:19:10',23,23.0,23,23,23,23.0,false,'23','23') +sql insert into tb2 values ('2021-05-05 18:19:11',24,24.0,24,24,24,24.0,false,'24','24') +sql insert into tb3 values ('2021-05-05 18:19:12',31,31.0,31,31,31,31.0,true ,'31','31') +sql insert into tb3 values ('2021-05-05 18:19:13',32,32.0,32,32,32,32.0,true ,'32','32') +sql insert into tb3 values ('2021-05-05 18:19:14',33,33.0,33,33,33,33.0,false,'33','33') +sql insert into tb3 values ('2021-05-05 18:19:15',34,34.0,34,34,34,34.0,false,'34','34') +sql insert into tb4 values ('2021-05-05 18:19:16',41,41.0,41,41,41,41.0,true ,'41','41') +sql insert into tb4 values ('2021-05-05 18:19:17',42,42.0,42,42,42,42.0,true ,'42','42') +sql insert into tb4 values ('2021-05-05 18:19:18',43,43.0,43,43,43,43.0,false,'43','43') +sql insert into tb4 values ('2021-05-05 18:19:19',44,44.0,44,44,44,44.0,false,'44','44') +sql insert into tb5 values ('2021-05-05 18:19:20',51,51.0,51,51,51,51.0,true ,'51','51') +sql insert into tb5 values ('2021-05-05 18:19:21',52,52.0,52,52,52,52.0,true ,'52','52') +sql insert into tb5 values ('2021-05-05 18:19:22',53,53.0,53,53,53,53.0,false,'53','53') +sql insert into tb5 values ('2021-05-05 18:19:23',54,54.0,54,54,54,54.0,false,'54','54') +sql insert into tb6 values ('2021-05-05 18:19:24',61,61.0,61,61,61,61.0,true ,'61','61') +sql insert into tb6 values ('2021-05-05 18:19:25',62,62.0,62,62,62,62.0,true ,'62','62') +sql insert into tb6 values ('2021-05-05 18:19:26',63,63.0,63,63,63,63.0,false,'63','63') +sql insert into tb6 values ('2021-05-05 18:19:27',64,64.0,64,64,64,64.0,false,'64','64') +sql insert into tb6 values ('2021-05-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb2 (ts timestamp, u1 int unsigned, u2 bigint unsigned, u3 smallint unsigned, u4 tinyint unsigned, ts2 timestamp) TAGS(t1 int unsigned, t2 bigint unsigned, t3 timestamp, t4 int) +sql create table tb2_1 using stb2 tags(1,1,'2021-05-05 18:38:38',1) +sql create table tb2_2 using stb2 tags(2,2,'2021-05-05 18:58:58',2) + +sql insert into tb2_1 values ('2021-05-05 18:19:00',1,2,3,4,'2021-05-05 18:28:01') +sql insert into tb2_1 values ('2021-05-05 18:19:01',5,6,7,8,'2021-05-05 18:28:02') +sql insert into tb2_1 values ('2021-05-05 18:19:02',2,2,3,4,'2021-05-05 18:28:03') +sql insert into tb2_1 values ('2021-05-05 18:19:03',5,6,7,8,'2021-05-05 18:28:04') +sql insert into tb2_1 values ('2021-05-05 18:19:04',3,2,3,4,'2021-05-05 18:28:05') +sql insert into tb2_1 values ('2021-05-05 18:19:05',5,6,7,8,'2021-05-05 18:28:06') +sql insert into tb2_1 values ('2021-05-05 18:19:06',4,2,3,4,'2021-05-05 18:28:07') +sql insert into tb2_1 values ('2021-05-05 18:19:07',5,6,7,8,'2021-05-05 18:28:08') +sql insert into tb2_1 values ('2021-05-05 18:19:08',5,2,3,4,'2021-05-05 18:28:09') +sql insert into tb2_1 values ('2021-05-05 18:19:09',5,6,7,8,'2021-05-05 18:28:10') +sql insert into tb2_1 values ('2021-05-05 18:19:10',6,2,3,4,'2021-05-05 18:28:11') +sql insert into tb2_2 values ('2021-05-05 18:19:11',5,6,7,8,'2021-05-05 18:28:12') +sql insert into tb2_2 values ('2021-05-05 18:19:12',7,2,3,4,'2021-05-05 18:28:13') +sql insert into tb2_2 values ('2021-05-05 18:19:13',5,6,7,8,'2021-05-05 18:28:14') +sql insert into tb2_2 values ('2021-05-05 18:19:14',8,2,3,4,'2021-05-05 18:28:15') +sql insert into tb2_2 values ('2021-05-05 18:19:15',5,6,7,8,'2021-05-05 18:28:16') + +sql create table stb3 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb3_1 using stb3 tags(1,'1',1.0) +sql create table tb3_2 using stb3 tags(2,'2',2.0) + +sql insert into tb3_1 values ('2021-01-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb3_1 values ('2021-02-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb3_1 values ('2021-03-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb3_1 values ('2021-04-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb3_1 values ('2021-05-05 18:19:28',5,NULL,5,NULL,5,NULL,true,NULL,'5') +sql insert into tb3_1 values ('2021-06-05 18:19:28',NULL,6.0,NULL,6,NULL,6.0,NULL,'6',NULL) +sql insert into tb3_1 values ('2021-07-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) +sql insert into tb3_2 values ('2021-01-06 18:19:00',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb3_2 values ('2021-02-06 18:19:01',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb3_2 values ('2021-03-06 18:19:02',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb3_2 values ('2021-04-06 18:19:03',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true,NULL,'15') +sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL) +sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb4 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9),c10 binary(16300)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb4_0 using stb4 tags(0,'0',0.0) +sql create table tb4_1 using stb4 tags(1,'1',1.0) +sql create table tb4_2 using stb4 tags(2,'2',2.0) +sql create table tb4_3 using stb4 tags(3,'3',3.0) +sql create table tb4_4 using stb4 tags(4,'4',4.0) + +$i = 0 +$ts0 = 1625850000000 +$blockNum = 5 +$delta = 0 +$tbname0 = tb4_ +$a = 0 +$b = 200 +$c = 400 +while $i < $blockNum + $x = 0 + $rowNum = 5 + while $x < $rowNum + $ts = $ts0 + $x + $a = $a + 1 + $b = $b + 1 + $c = $c + 1 + $d = $x / 10 + $tin = $rowNum + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + $tbname = 'tb4_ . $i + $tbname = $tbname . ' + sql insert into $tbname values ( $ts , $a , $b , $c , $d , $d , $c , true, $binary , $nchar , $binary ) + $x = $x + 1 + endw + + $i = $i + 1 + $ts0 = $ts0 + 259200000 +endw + + +print ============== query +sql select a.ts,a.c1,a.c8 from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and ((a.c1 > 50 and a.c1 < 60) or (b.c2 > 60));; +sql select * from stb1 where (c6 > 3.0 or c6 < 60) and c6 > 50 and (c6 != 53 or c6 != 63);; +sql select ts,c1 from stb1 where (c1 > 60 or c1 < 10 or (c1 > 20 and c1 < 30)) and ts > '2021-05-05 18:19:00.000' and ts < '2021-05-05 18:19:25.000' and c1 != 21 and c1 != 22 order by ts; +sql select a.* from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and a.c1 > 50 order by ts;; +sql select a.ts,b.ts,a.c1,b.u1,b.u2 from (select * from stb1) a, (select * from stb2) b where a.ts=b.ts and (a.c1 < 10 or a.c1 > 30) and (b.u1 < 5 or b.u1 > 5) order by ts;; +sql select a.ts,b.ts,a.c1,b.u1,b.u2 from (select * from stb1) a, (select * from stb2) b where a.ts=b.ts and a.c1 < 30 and b.u1 > 1 and a.c1 > 10 and b.u1 < 8 and b.u1<>5 order by ts;; +sql select tb1.ts,tb1.*,tb2_1.* from tb1, tb2_1 where tb1.ts=tb2_1.ts and tb1.ts >= '2021-05-05 18:19:03.000' and tb1.c7=false and tb2_1.u3>4 order by ts;; +sql select stb1.ts,stb1.c1,stb1.t1,stb2.ts,stb2.u1,stb2.t4 from stb1, stb2 where stb1.ts=stb2.ts and stb1.t1 = stb2.t4 order by ts;; +sql select count(*) from stb1 where tbname like 'tb%' or c1 > 0;; +sql select * from stb1 where tbname like 'tb%' and (t1=1 or t2=2 or t3=3) and t1 > 2 order by ts;; + +_OVER: +system sh/exec.sh -n dnode1 -s stop -x SIGINT +print =============== check +$null= + +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content > 2 then + return -1 +endi + +if $system_content == $null then + return -1 +endi From cfc856ac5346cbde4b4507ac8a6d39c773c30746 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 9 Aug 2022 20:03:08 +0800 Subject: [PATCH 092/165] feat: update taostools for3.0 (#15914) * feat: update taos-tools for 3.0 [TD-14141] * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools 8e3b3ee * fix: remove submodules * feat: update taos-tools c529299 * feat: update taos-tools 9dc2fec for 3.0 * fix: optim upx * feat: update taos-tools f4e456a for 3.0 * feat: update taos-tools 2a2def1 for 3.0 * feat: update taos-tools c9cc20f for 3.0 * feat: update taostoosl 8a5e336 for 3.0 * feat: update taostools 3c7dafe for 3.0 * feat: update taos-tools 2d68404 for 3.0 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9ee0ea526c..4f08656ac8 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 3c7dafe + GIT_TAG 2d68404 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 3e626d56d15978b39ff3ea835fe21593719aa97c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 9 Aug 2022 20:32:02 +0800 Subject: [PATCH 093/165] fix: set none if cols not full --- source/libs/executor/src/dataInserter.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 8e355ad3df..26e1f0a3ee 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -213,6 +213,9 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); } } + if(!fullCol) { + rb.hasNone = true; + } tdSRowEnd(&rb); if (ignoreRow) { From d74732c16b437000eb0365837a7e58c9e9ed99ef Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 9 Aug 2022 20:36:56 +0800 Subject: [PATCH 094/165] fix: modify udf document --- docs/zh/07-develop/09-udf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 08da9e296c..61b4974451 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -13,7 +13,7 @@ TDengine 支持通过 C/C++ 语言进行 UDF 定义。接下来结合示例讲 实现 UDF 时，需要实现规定的接口函数 - 标量函数需要实现标量接口函数 scalarfn 。 - 聚合函数需要实现聚合接口函数 aggfn_start ， aggfn ， aggfn_finish。 -- 如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destory。 +- 如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destroy。 接口函数的名称是 UDF 名称，或者是 UDF 名称和特定后缀（_start, _finish, _init, _destroy)的连接。列表中的scalarfn，aggfn, udf需要替换成udf函数名。 From 8648fe6579bfcf6c678f4a13f03c07aba20d0b12 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 9 Aug 2022 20:43:02 +0800 Subject: [PATCH 095/165] build: chmod build.sh --- build.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build.sh diff --git a/build.sh b/build.sh old mode 100644 new mode 100755 From 7df0a65e75e427d1a0df06c6b2929e48ad072d75 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 20:57:34 +0800 Subject: [PATCH 096/165] fix(query): set correct meta size in handling json. --- source/client/src/clientImpl.c | 12 ++++++++++-- tests/script/tsim/compute/diff2.sim | 1 - 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7838d87a20..d3079a48d7 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1571,10 +1571,18 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int return TSDB_CODE_SUCCESS; } +static int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { + int32_t cols = *(int32_t*) (p + sizeof(int32_t) * 3); + ASSERT(numOfCols == cols); + + return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t)*3 + sizeof(uint64_t) + numOfCols * (sizeof(int8_t) + sizeof(int32_t)); +} + static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { char* p = (char*)pResultInfo->pData; - int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + // version + length + numOfRows + numOfCol + groupId + flag_segment + column_info + int32_t len = getVersion1BlockMetaSize(p, numOfCols); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; @@ -1642,7 +1650,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* p1 = pResultInfo->convertJson; int32_t totalLen = 0; - int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + int32_t len = getVersion1BlockMetaSize(p, numOfCols); memcpy(p1, p, len); p += len; diff --git a/tests/script/tsim/compute/diff2.sim b/tests/script/tsim/compute/diff2.sim index a09bee991e..5a2c3706ee 100644 --- a/tests/script/tsim/compute/diff2.sim +++ b/tests/script/tsim/compute/diff2.sim @@ -78,7 +78,6 @@ endi sql select diff(c7) from $tb sql_error select diff(c8) from $tb sql_error select diff(c9) from $tb -sql_error select diff(ts) from $tb sql select diff(c1), diff(c2) from $tb sql select 2+diff(c1) from $tb From ae7b2954f4918c5f6e7fe6bf07b91c706fa688b1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 21:02:24 +0800 Subject: [PATCH 097/165] test: remove invalid cases. --- tests/system-test/2-query/diff.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index 76d4891a1c..ceef3f93eb 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -83,9 +83,6 @@ class TDTestCase: tdSql.execute(f"insert into {dbname}.stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - tdSql.error(f"select diff(ts) from {dbname}.stb") - tdSql.error(f"select diff(ts) from {dbname}.stb_1") - # tdSql.error(f"select diff(col7) from {dbname}.stb") tdSql.error(f"select diff(col8) from {dbname}.stb") From 597d360d853c1479c76a224a1daee239a6da5314 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Aug 2022 21:31:22 +0800 Subject: [PATCH 098/165] test: remove invalid cases. --- tests/system-test/2-query/function_diff.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/system-test/2-query/function_diff.py b/tests/system-test/2-query/function_diff.py index 2f463e59a0..492703d3ec 100644 --- a/tests/system-test/2-query/function_diff.py +++ b/tests/system-test/2-query/function_diff.py @@ -256,8 +256,6 @@ class TDTestCase: tdSql.error(self.diff_query_form(col='c%')) # col is spercial char 1 tdSql.error(self.diff_query_form(col='c_')) # col is spercial char 2 tdSql.error(self.diff_query_form(col='c.')) # col is spercial char 3 - tdSql.error(self.diff_query_form(col='c3')) # timestamp col - tdSql.error(self.diff_query_form(col='ts')) # Primary key tdSql.error(self.diff_query_form(col='avg(c1)')) # expr col # tdSql.error(self.diff_query_form(col='c6')) # bool col tdSql.query("select diff(c6) from t1") From fc225a580dd5eb6a6af5e9943d8304152a6b6014 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 9 Aug 2022 21:37:21 +0800 Subject: [PATCH 099/165] enh(stream): recover process --- include/libs/stream/tstream.h | 33 ++--- source/libs/stream/inc/streamInc.h | 2 +- source/libs/stream/src/stream.c | 32 +++-- source/libs/stream/src/streamExec.c | 34 +++--- source/libs/stream/src/streamRecover.c | 159 ++++++++++++++++++++++++- 5 files changed, 204 insertions(+), 56 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 8aacfec397..9355d76dcb 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -154,7 +154,10 @@ static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) { atomic_store_8(&queue->status, STREAM_QUEUE__FAILED); } -static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; } +static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { + // + return queue->qItem; +} static FORCE_INLINE void* streamQueueNextItem(SStreamQueue* queue) { int8_t dequeueFlag = atomic_exchange_8(&queue->status, STREAM_QUEUE__PROCESSING); @@ -226,9 +229,7 @@ typedef struct { int32_t nodeId; int32_t childId; int32_t taskId; - // int64_t checkpointVer; - // int64_t processedVer; - SEpSet epSet; + SEpSet epSet; } SStreamChildEpInfo; typedef struct { @@ -372,15 +373,6 @@ static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBloc return 0; } -typedef struct { - int32_t reserved; -} SStreamTaskDeployRsp; - -typedef struct { - // SMsgHead head; - SStreamTask* task; -} SStreamTaskDeployReq; - typedef struct { SMsgHead head; int64_t streamId; @@ -478,7 +470,18 @@ typedef struct { } SStreamRecoverDownstreamRsp; int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq); -int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, const SStreamRecoverDownstreamRsp* pRsp); +int32_t tDecodeSStreamTaskRecoverReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq); + +int32_t tEncodeSStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamRecoverDownstreamRsp* pRsp); +int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamRecoverDownstreamRsp* pRsp); + +typedef struct { + int64_t streamId; + int32_t taskId; + int32_t waitingRspCnt; + int32_t totReq; + SArray* info; // SArray*> +} SStreamRecoverStatus; int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); @@ -504,7 +507,7 @@ typedef struct SStreamMeta { TTB* pTaskDb; TTB* pStateDb; SHashObj* pTasks; - SHashObj* pRecoveringState; + SHashObj* pRecoverStatus; void* ahandle; TXN txn; FTaskExpand* expandFunc; diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 1ff27f1253..7dc8d822e9 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -33,7 +33,7 @@ typedef struct { static SStreamGlobalEnv streamEnv; int32_t streamExec(SStreamTask* pTask); -int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum); +int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch); int32_t streamDispatch(SStreamTask* pTask); int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 1f8d742de4..6da7d4fd59 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -104,7 +104,7 @@ int32_t streamSchedExec(SStreamTask* pTask) { return 0; } -int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp) { +int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) { SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); int8_t status; @@ -136,7 +136,6 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp->pCont = buf; pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); tmsgSendRsp(pRsp); - tFreeStreamDispatchReq(pReq); return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; } @@ -183,6 +182,7 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S pReq->upstreamTaskId); streamTaskEnqueue(pTask, pReq, pRsp); + tFreeStreamDispatchReq(pReq); if (exec) { streamTryExec(pTask); @@ -246,24 +246,20 @@ int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, return 0; } -int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) { - if (pRsp->inputStatus == TASK_INPUT_STATUS__NORMAL) { - pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; +int32_t streamProcessRecoverRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamRecoverDownstreamRsp* pRsp) { + streamProcessRunReq(pTask); - streamProcessRunReq(pTask); - - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - // scan data to recover - pTask->inputStatus = TASK_INPUT_STATUS__RECOVER; - pTask->taskStatus = TASK_STATUS__RECOVERING; - qStreamPrepareRecover(pTask->exec.executor, pTask->startVer, pTask->recoverSnapVer); - if (streamPipelineExec(pTask, 100) < 0) { - return -1; - } - } else { - pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; - pTask->taskStatus = TASK_STATUS__NORMAL; + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + // scan data to recover + pTask->inputStatus = TASK_INPUT_STATUS__RECOVER; + pTask->taskStatus = TASK_STATUS__RECOVER_SELF; + qStreamPrepareRecover(pTask->exec.executor, pTask->startVer, pTask->recoverSnapVer); + if (streamPipelineExec(pTask, 100, true) < 0) { + return -1; } + } else { + pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; + pTask->taskStatus = TASK_STATUS__NORMAL; } return 0; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index ffb7c04bf2..72249c5181 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -93,7 +93,7 @@ static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock } #endif -int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { +int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch) { ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); void* exec = pTask->exec.executor; @@ -125,24 +125,26 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { taosArrayDestroy(pRes); break; } - SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); - if (qRes == NULL) { - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - return -1; - } + if (dispatch) { + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + if (qRes == NULL) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + return -1; + } - qRes->type = STREAM_INPUT__DATA_BLOCK; - qRes->blocks = pRes; - qRes->childId = pTask->selfChildId; + qRes->type = STREAM_INPUT__DATA_BLOCK; + qRes->blocks = pRes; + qRes->childId = pTask->selfChildId; - if (streamTaskOutput(pTask, qRes) < 0) { - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - taosFreeQitem(qRes); - return -1; - } + if (streamTaskOutput(pTask, qRes) < 0) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + taosFreeQitem(qRes); + return -1; + } - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - streamDispatch(pTask); + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + streamDispatch(pTask); + } } } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index b2a7e00877..263053778b 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -132,6 +132,49 @@ int32_t tDecodeSStreamMultiVgCheckpointInfo(SDecoder* pDecoder, SStreamMultiVgCh return 0; } +int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq) { + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + return 0; +} + +int32_t tDecodeSStreamTaskRecoverReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq) { + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + return 0; +} + +int32_t tEncodeSStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamRecoverDownstreamRsp* pRsp) { + if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1; + int32_t sz = taosArrayGetSize(pRsp->checkpointVer); + if (tEncodeI32(pEncoder, sz) < 0) return -1; + for (int32_t i = 0; i < sz; i++) { + SStreamCheckpointInfo* pInfo = taosArrayGet(pRsp->checkpointVer, i); + if (tEncodeSStreamCheckpointInfo(pEncoder, pInfo) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamRecoverDownstreamRsp* pRsp) { + if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->taskId) < 0) return -1; + int32_t sz; + if (tDecodeI32(pDecoder, &sz) < 0) return -1; + pRsp->checkpointVer = taosArrayInit(sz, sizeof(SStreamCheckpointInfo)); + if (pRsp->checkpointVer == NULL) return -1; + for (int32_t i = 0; i < sz; i++) { + SStreamCheckpointInfo info; + if (tDecodeSStreamCheckpointInfo(pDecoder, &info) < 0) return -1; + taosArrayPush(pRsp->checkpointVer, &info); + } + return 0; +} + int32_t streamSaveStateInfo(SStreamMeta* pMeta, SStreamTask* pTask) { void* buf = NULL; @@ -223,25 +266,129 @@ int32_t streamSaveAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) { return 0; } -int32_t streamFetchDownstreamStatus(SStreamTask* pTask) { +int32_t streamFetchRecoverStatus(SStreamTask* pTask, const SVgroupInfo* pVgInfo) { + int32_t taskId = pVgInfo->taskId; + int32_t nodeId = pVgInfo->vgId; + SStreamRecoverDownstreamReq req = { + .streamId = pTask->taskId, + .downstreamTaskId = taskId, + .taskId = pTask->taskId, + }; + int32_t tlen; + int32_t code; + tEncodeSize(tEncodeSStreamTaskRecoverReq, &req, tlen, code); + if (code < 0) { + return -1; + } + void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + tlen); + if (buf == NULL) { + return -1; + } + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + SEncoder encoder; + tEncoderInit(&encoder, abuf, tlen); + if (tEncodeSStreamTaskRecoverReq(&encoder, &req) < 0) { + tEncoderClear(&encoder); + taosMemoryFree(buf); + return -1; + } + tEncoderClear(&encoder); + + ((SMsgHead*)buf)->vgId = htonl(nodeId); + SRpcMsg msg = { + .pCont = buf, .contLen = sizeof(SMsgHead) + tlen, + /*.msgType = */ + }; + tmsgSendReq(&pVgInfo->epSet, &msg); + + return 0; +} + +int32_t streamFetchDownstreamStatus(SStreamMeta* pMeta, SStreamTask* pTask) { // set self status to recover_phase1 - // build fetch status msg - // send fetch msg + SStreamRecoverStatus* pRecover; atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_DOWNSTREAM); + pRecover = taosHashGet(pMeta->pRecoverStatus, &pTask->taskId, sizeof(int32_t)); + if (pRecover == NULL) { + pRecover = taosMemoryCalloc(1, sizeof(SStreamRecoverStatus)); + if (pRecover == NULL) { + return -1; + } + pRecover->info = taosArrayInit(0, sizeof(void*)); + if (pRecover->info == NULL) { + taosMemoryFree(pRecover); + return -1; + } + taosHashPut(pMeta->pRecoverStatus, &pTask->taskId, sizeof(int32_t), &pRecover, sizeof(void*)); + } if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + pRecover->totReq = 1; } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + int32_t numOfDownstream = taosArrayGetSize(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); + pRecover->totReq = numOfDownstream; + for (int32_t i = 0; i < numOfDownstream; i++) { + SVgroupInfo* pVgInfo = taosArrayGet(pTask->shuffleDispatcher.dbInfo.pVgroupInfos, i); + streamFetchRecoverStatus(pTask, pVgInfo); + } } else { ASSERT(0); } return 0; } -int32_t streamProcessFetchStatusRsp(SStreamMeta* pMeta, SStreamTask* pTask, void* msg) { +int32_t streamProcessFetchStatusRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamRecoverDownstreamRsp* pRsp) { // if failed, set timer and retry // if successful - // add rsp state to partial recover hash - // if complete, begin actual recover + int32_t taskId = pTask->taskId; + SStreamRecoverStatus* pRecover = taosHashGet(pMeta->pRecoverStatus, &taskId, sizeof(int32_t)); + if (pRecover == NULL) { + return -1; + } + + taosArrayPush(pRecover->info, &pRsp->checkpointVer); + + int32_t leftRsp = atomic_sub_fetch_32(&pRecover->waitingRspCnt, 1); + ASSERT(leftRsp >= 0); + + if (leftRsp == 0) { + ASSERT(taosArrayGetSize(pRecover->info) == pRecover->totReq); + + // srcNodeId -> SStreamCheckpointInfo* + SHashObj* pFinalChecks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (pFinalChecks == NULL) return -1; + + for (int32_t i = 0; i < pRecover->totReq; i++) { + SArray* pChecks = taosArrayGetP(pRecover->info, i); + int32_t sz = taosArrayGetSize(pChecks); + for (int32_t j = 0; j < sz; j++) { + SStreamCheckpointInfo* pOneCheck = taosArrayGet(pChecks, j); + SStreamCheckpointInfo* pCheck = taosHashGet(pFinalChecks, &pOneCheck->srcNodeId, sizeof(int32_t)); + if (pCheck == NULL) { + pCheck = taosMemoryCalloc(1, sizeof(SStreamCheckpointInfo)); + pCheck->srcNodeId = pOneCheck->srcNodeId; + pCheck->srcChildId = pOneCheck->srcChildId; + pCheck->stateProcessedVer = pOneCheck->stateProcessedVer; + taosHashPut(pFinalChecks, &pCheck->srcNodeId, sizeof(int32_t), &pCheck, sizeof(void*)); + } else { + pCheck->stateProcessedVer = TMIN(pCheck->stateProcessedVer, pOneCheck->stateProcessedVer); + } + } + } + // load local state + // + // recover + // + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + qStreamPrepareRecover(pTask->exec.executor, pTask->startVer, pTask->recoverSnapVer); + if (streamPipelineExec(pTask, 10000, true) < 0) { + return -1; + } + } + taosHashCleanup(pFinalChecks); + taosHashRemove(pMeta->pRecoverStatus, &taskId, sizeof(int32_t)); + atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + } return 0; } From 3f0c3076a93ab8529a0ccd358a9877e9535cbf84 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 9 Aug 2022 22:29:54 +0800 Subject: [PATCH 100/165] test: open sim testcase of sync --- tests/script/jenkins/basic.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 23676153b5..0bb24826a3 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -332,8 +332,8 @@ ./test.sh -f tsim/vnode/stable_replica3_vnode3.sim # --- sync -#./test.sh -f tsim/sync/3Replica1VgElect.sim -#./test.sh -f tsim/sync/3Replica5VgElect.sim +./test.sh -f tsim/sync/3Replica1VgElect.sim +./test.sh -f tsim/sync/3Replica5VgElect.sim ./test.sh -f tsim/sync/oneReplica1VgElect.sim ./test.sh -f tsim/sync/oneReplica5VgElect.sim From 8665f8056df482ae9e07d4d5b304debf9389def8 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 00:21:42 +0800 Subject: [PATCH 101/165] build: release ver-3.0.0.10003 --- packaging/release.bat | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/release.bat b/packaging/release.bat index d58e19cece..ffd3a68048 100644 --- a/packaging/release.bat +++ b/packaging/release.bat @@ -11,13 +11,13 @@ if !%2==! GOTO USAGE if "%1" == "cluster" ( set work_dir=%internal_dir% set packagServerName_x64=TDengine-enterprise-server-%2-beta-Windows-x64 - set packagServerName_x86=TDengine-enterprise-server-%2-beta-Windows-x86 + @REM set packagServerName_x86=TDengine-enterprise-server-%2-beta-Windows-x86 set packagClientName_x64=TDengine-enterprise-client-%2-beta-Windows-x64 set packagClientName_x86=TDengine-enterprise-client-%2-beta-Windows-x86 ) else ( set work_dir=%community_dir% set packagServerName_x64=TDengine-server-%2-Windows-x64 - set packagServerName_x86=TDengine-server-%2-Windows-x86 + @REM set packagServerName_x86=TDengine-server-%2-Windows-x86 set packagClientName_x64=TDengine-client-%2-Windows-x64 set packagClientName_x86=TDengine-client-%2-Windows-x86 ) @@ -59,8 +59,8 @@ rd /s /Q C:\TDengine cmake --install . if not %errorlevel% == 0 ( call :RUNFAILED build x86 failed & exit /b 1) cd %package_dir% -iscc /DMyAppInstallName="%packagServerName_x86%" /DMyAppVersion="%2" /DMyAppExcludeSource="" tools\tdengine.iss /O..\release -if not %errorlevel% == 0 ( call :RUNFAILED package %packagServerName_x86% failed & exit /b 1) +@REM iscc /DMyAppInstallName="%packagServerName_x86%" /DMyAppVersion="%2" /DMyAppExcludeSource="" tools\tdengine.iss /O..\release +@REM if not %errorlevel% == 0 ( call :RUNFAILED package %packagServerName_x86% failed & exit /b 1) iscc /DMyAppInstallName="%packagClientName_x86%" /DMyAppVersion="%2" /DMyAppExcludeSource="taosd.exe" tools\tdengine.iss /O..\release if not %errorlevel% == 0 ( call :RUNFAILED package %packagClientName_x86% failed & exit /b 1) From 3fe50ef41e56c82b5af79e52d70c6c47bebaebfb Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 00:29:08 +0800 Subject: [PATCH 102/165] build: release ver-3.0.0.10003 --- cmake/cmake.platform | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index cd1be78fbc..887fbd86d5 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -97,13 +97,13 @@ IF ("${CPUTYPE}" STREQUAL "") ELSE () # if generate ARM version: # cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64 - IF (${CPUTYPE} MATCHES "aarch32" or ${CPUTYPE} MATCHES "arm32") + IF (${CPUTYPE} MATCHES "aarch32" OR ${CPUTYPE} MATCHES "arm32") SET(PLATFORM_ARCH_STR "arm") MESSAGE(STATUS "input cpuType: aarch32") ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_32") SET(TD_ARM_32 TRUE) - ELSEIF (${CPUTYPE} MATCHES "aarch64" or ${CPUTYPE} MATCHES "arm64") + ELSEIF (${CPUTYPE} MATCHES "aarch64" OR ${CPUTYPE} MATCHES "arm64") SET(PLATFORM_ARCH_STR "arm64") MESSAGE(STATUS "input cpuType: aarch64") ADD_DEFINITIONS("-D_TD_ARM_") From fe962c0d16f834b532427b3c6e084f1215e7fa1e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 10 Aug 2022 09:07:48 +0800 Subject: [PATCH 103/165] fix: fix ntables error in database --- source/common/src/systable.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 2a7c6ca471..ce80942356 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -74,7 +74,7 @@ static const SSysDbTableSchema clusterSchema[] = { static const SSysDbTableSchema userDBSchema[] = { {.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "vgroups", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "replica", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "strict", .bytes = TSDB_DB_STRICT_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 8edfcad2c6..89ea7abc23 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1706,7 +1706,10 @@ static void setPerfSchemaDbCfg(SDbObj *pDbObj) { static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; int32_t *numOfTables = p1; - *numOfTables += pVgroup->numOfTables; + int64_t uid = *(int64_t*)p2; + if (pVgroup->dbUid == uid) { + *numOfTables += pVgroup->numOfTables; + } return true; } @@ -1747,7 +1750,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) { int32_t numOfTables = 0; - sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL); + sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, &pDb->uid, NULL); mndDumpDbInfoData(pMnode, pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus, sysinfo); numOfRows++; } From 38807616e6e493a45c7db0736f75d00c2eca959f Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 10:29:47 +0800 Subject: [PATCH 104/165] build: release ver-3.0.0.10003 --- cmake/cmake.install | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/cmake.install b/cmake/cmake.install index d9f759217a..4b1ccbf6d5 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -22,6 +22,7 @@ ELSEIF (TD_WINDOWS) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosd.exe DESTINATION .) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/udfd.exe DESTINATION .) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosBenchmark.exe DESTINATION .) IF (TD_MVN_INSTALLED) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.38-dist.jar DESTINATION connector/jdbc) From 67fbb7391031e561b6bfe633e18b67de273f6a21 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 11:23:37 +0800 Subject: [PATCH 105/165] build: release ver-3.0.0.10003 --- include/os/osSystem.h | 1 + source/os/src/osSysinfo.c | 12 +++++++++++- source/os/src/osSystem.c | 10 ++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 6770be6e46..c5b8fc7be7 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -32,6 +32,7 @@ extern "C" { typedef struct TdCmd *TdCmdPtr; TdCmdPtr taosOpenCmd(const char* cmd); +int64_t taosGetsCmd(TdCmdPtr pCmd, int32_t maxSize, char *__restrict buf); int64_t taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf); int32_t taosEOFCmd(TdCmdPtr pCmd); int64_t taosCloseCmd(TdCmdPtr* ppCmd); diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 3e68b6e086..3a75e18a7f 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -398,7 +398,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); - if (code != 0) { + if (code != 0 && (done & 1) == 0) { TdFilePtr pFile1 = taosOpenFile("/proc/device-tree/model", TD_FILE_READ | TD_FILE_STREAM); if (pFile1 == NULL) return code; taosGetsFile(pFile1, maxLen, cpuModel); @@ -407,6 +407,16 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { done |= 1; } + if (code != 0 && (done & 1) == 0) { + TdCmdPtr pCmd = taosOpenCmd("uname -a"); + if (pCmd == NULL) return code; + if (taosGetsCmd(pCmd, maxLen, cpuModel) > 0) { + code = 0; + done |= 1; + } + taosCloseCmd(&pCmd); + } + if ((done & 2) == 0) { *numOfCores = coreCount; done |= 2; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index c86cd19e32..b6f6637601 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -248,6 +248,16 @@ TdCmdPtr taosOpenCmd(const char* cmd) { #endif } +int64_t taosGetsCmd(TdCmdPtr pCmd, int32_t maxSize, char *__restrict buf) { + if (pCmd == NULL || buf == NULL) { + return -1; + } + if (fgets(buf, maxSize, (FILE*)pCmd) == NULL) { + return -1; + } + return strlen(buf); +} + int64_t taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf) { if (pCmd == NULL || ptrBuf == NULL) { return -1; From 712f7e8cce85efdf79727f8ffc847a6c62b0e7e5 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 11:47:45 +0800 Subject: [PATCH 106/165] os: fix copy appname overflow --- source/os/src/osSemaphore.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 63c414386b..5da78f002f 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -57,7 +57,7 @@ int32_t taosGetAppName(char* name, int32_t* len) { end = filepath; } - strcpy(name, end); + tstrncpy(name, end, TSDB_APP_NAME_LEN); if (len != NULL) { *len = (int32_t)strlen(end); @@ -625,7 +625,7 @@ int32_t taosGetAppName(char *name, int32_t *len) { buf[PATH_MAX] = '\0'; size_t n = strlen(buf); if (len) *len = n; - if (name) strcpy(name, buf); + if (name) tstrncpy(name, buf, TSDB_APP_NAME_LEN); return 0; } @@ -668,7 +668,7 @@ int32_t taosGetAppName(char* name, int32_t* len) { ++end; - strcpy(name, end); + tstrncpy(name, end, TSDB_APP_NAME_LEN); if (len != NULL) { *len = strlen(name); From 59d982333cd52e6042d5fd0c4a4f3fc4b4fff3c4 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 13:05:46 +0800 Subject: [PATCH 107/165] os: fix copy appname overflow --- source/os/src/osSemaphore.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 5da78f002f..c57ea33028 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -16,6 +16,7 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" +#include "tdef.h" #include "pthread.h" #ifdef WINDOWS From abc90650944c782b23851f09fac1c7b4f81e94d0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 13:13:25 +0800 Subject: [PATCH 108/165] fix(query): set correct raw block info. --- source/client/inc/clientInt.h | 6 ++---- source/client/src/clientImpl.c | 2 +- source/client/src/tmq.c | 2 +- source/libs/executor/src/executorimpl.c | 1 - source/libs/executor/src/sortoperator.c | 2 +- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index e7a099cf23..f275ae0885 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -101,10 +101,6 @@ typedef struct SQueryExecMetric { int64_t rsp; // receive response from server, us } SQueryExecMetric; -typedef struct SHeartBeatInfo { - void* pTimer; // timer, used to send request msg to mnode -} SHeartBeatInfo; - struct SAppInstInfo { int64_t numOfConns; SCorEpSet mgmtEp; @@ -256,6 +252,8 @@ SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool valida TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly); void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); +int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols); + static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; return (SReqResultInfo*)&msg->resInfo; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d3079a48d7..5c37e6dc15 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1571,7 +1571,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int return TSDB_CODE_SUCCESS; } -static int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { +int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { int32_t cols = *(int32_t*) (p + sizeof(int32_t) * 3); ASSERT(numOfCols == cols); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index e2619a78b1..8f866bde8e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -2947,7 +2947,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; - char* pStart = pData + sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols) - numOfCols * (sizeof(int8_t) + sizeof(int32_t)); int32_t* colLength = (int32_t*)pStart; pStart += sizeof(int32_t) * numOfCols; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5f5c035523..d3ced810c6 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2157,7 +2157,6 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn char* pStart = pRetrieveRsp->data; while(index++ < pRetrieveRsp->numOfBlocks) { SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); - blockDataEnsureCapacity(pb, pRetrieveRsp->numOfRows); code = extractDataBlockFromFetchRsp(pb, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 16f35b1b0d..4dd5e4ec15 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -658,7 +658,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataDestroy(p); - qDebug("%s get sorted row blocks, rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.rows); + qDebug("%s get sorted block, groupId:%0x"PRIx64" rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } From 10266ec7fa1255b2b8b13c7a18c8b4b6d1c84a82 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 10 Aug 2022 13:47:38 +0800 Subject: [PATCH 109/165] docs: update k8s doc for 3.0 (#15852) * docs: update k8s doc for 3.0 * docs: update 03-k8s.md --- docs/zh/10-deployment/03-k8s.md | 327 ++++++++++++++------------------ 1 file changed, 140 insertions(+), 187 deletions(-) diff --git a/docs/zh/10-deployment/03-k8s.md b/docs/zh/10-deployment/03-k8s.md index d45a3b8030..ab1bff4449 100644 --- a/docs/zh/10-deployment/03-k8s.md +++ b/docs/zh/10-deployment/03-k8s.md @@ -3,27 +3,11 @@ sidebar_label: Kubernetes title: 在 Kubernetes 上部署 TDengine 集群 --- -## 配置 ConfigMap +以下配置文件可以从 [GitHub 仓库](https://github.com/taosdata/TDengine-Operator/tree/3.0/src/tdengine) 下载。 -为 TDengine 创建 `taoscfg.yaml`，此文件中的配置将作为环境变量传入 TDengine 镜像，更新此配置将导致所有 TDengine POD 重启。 +## 配置 Service 服务 -```yaml ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: taoscfg - labels: - app: tdengine -data: - CLUSTER: "1" - TAOS_KEEP: "3650" - TAOS_DEBUG_FLAG: "135" -``` - -## 配置服务 - -创建一个 service 配置文件：`taosd-service.yaml`，服务名称 `metadata.name` (此处为 "taosd") 将在下一步中使用到。添加 TDengine 所用到的所有端口： +创建一个 Service 配置文件：`taosd-service.yaml`，服务名称 `metadata.name` (此处为 "taosd") 将在下一步中使用到。添加 TDengine 所用到的所有端口： ```yaml --- @@ -38,45 +22,9 @@ spec: - name: tcp6030 protocol: "TCP" port: 6030 - - name: tcp6035 - protocol: "TCP" - port: 6035 - name: tcp6041 protocol: "TCP" port: 6041 - - name: udp6030 - protocol: "UDP" - port: 6030 - - name: udp6031 - protocol: "UDP" - port: 6031 - - name: udp6032 - protocol: "UDP" - port: 6032 - - name: udp6033 - protocol: "UDP" - port: 6033 - - name: udp6034 - protocol: "UDP" - port: 6034 - - name: udp6035 - protocol: "UDP" - port: 6035 - - name: udp6036 - protocol: "UDP" - port: 6036 - - name: udp6037 - protocol: "UDP" - port: 6037 - - name: udp6038 - protocol: "UDP" - port: 6038 - - name: udp6039 - protocol: "UDP" - port: 6039 - - name: udp6040 - protocol: "UDP" - port: 6040 selector: app: "tdengine" ``` @@ -109,7 +57,7 @@ spec: spec: containers: - name: "tdengine" - image: "zitsen/taosd:develop" + image: "tdengine/tdengine:3.0.0.0" imagePullPolicy: "Always" envFrom: - configMapRef: @@ -118,45 +66,9 @@ spec: - name: tcp6030 protocol: "TCP" containerPort: 6030 - - name: tcp6035 - protocol: "TCP" - containerPort: 6035 - name: tcp6041 protocol: "TCP" containerPort: 6041 - - name: udp6030 - protocol: "UDP" - containerPort: 6030 - - name: udp6031 - protocol: "UDP" - containerPort: 6031 - - name: udp6032 - protocol: "UDP" - containerPort: 6032 - - name: udp6033 - protocol: "UDP" - containerPort: 6033 - - name: udp6034 - protocol: "UDP" - containerPort: 6034 - - name: udp6035 - protocol: "UDP" - containerPort: 6035 - - name: udp6036 - protocol: "UDP" - containerPort: 6036 - - name: udp6037 - protocol: "UDP" - containerPort: 6037 - - name: udp6038 - protocol: "UDP" - containerPort: 6038 - - name: udp6039 - protocol: "UDP" - containerPort: 6039 - - name: udp6040 - protocol: "UDP" - containerPort: 6040 env: # POD_NAME for FQDN config - name: POD_NAME @@ -190,14 +102,13 @@ spec: readinessProbe: exec: command: - - taos - - -s - - "show mnodes" + - taos-check initialDelaySeconds: 5 timeoutSeconds: 5000 livenessProbe: - tcpSocket: - port: 6030 + exec: + command: + - taos-check initialDelaySeconds: 15 periodSeconds: 20 volumeClaimTemplates: @@ -206,44 +117,78 @@ spec: spec: accessModes: - "ReadWriteOnce" - storageClassName: "csi-rbd-sc" + storageClassName: "standard" resources: requests: storage: "10Gi" ``` -## 启动集群 +## 使用 kubectl 命令部署 TDengine 集群 -将前述三个文件添加到 Kubernetes 集群中： +顺序执行以下命令。 ```bash -kubectl apply -f taoscfg.yaml kubectl apply -f taosd-service.yaml kubectl apply -f tdengine.yaml ``` -上面的配置将生成一个两节点的 TDengine 集群，dnode 是自动配置的，可以使用 `show dnodes` 命令查看当前集群的节点： +上面的配置将生成一个三节点的 TDengine 集群，dnode 是自动配置的，可以使用 show dnodes 命令查看当前集群的节点： ```bash kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" kubectl exec -i -t tdengine-1 -- taos -s "show dnodes" - +kubectl exec -i -t tdengine-2 -- taos -s "show dnodes" ``` 输出如下： ``` -Welcome to the TDengine shell from Linux, Client Version:2.1.1.0 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 17:13:24.181 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 17:14:09.257 | | -Query OK, 2 row(s) in set (0.000997s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | + 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | + 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | +Query OK, 3 rows in database (0.003655s) +``` +## 使能端口转发 + +利用 kubectl 端口转发功能可以使应用可以访问 Kubernetes 环境运行的 TDengine 集群。 + +``` +kubectl port-forward tdengine-0 6041:6041 & +``` + +使用 curl 命令验证 TDengine REST API 使用的 6041 接口。 + +``` +$ curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql +Handling connection for 6041 +{"code":0,"column_meta":[["name","VARCHAR",64],["create_time","TIMESTAMP",8],["vgroups","SMALLINT",2],["ntables","BIGINT",8],["replica","TINYINT",1],["strict","VARCHAR",4],["duration","VARCHAR",10],["keep","VARCHAR",32],["buffer","INT",4],["pagesize","INT",4],["pages","INT",4],["minrows","INT",4],["maxrows","INT",4],["comp","TINYINT",1],["precision","VARCHAR",2],["status","VARCHAR",10],["retention","VARCHAR",60],["single_stable","BOOL",1],["cachemodel","VARCHAR",11],["cachesize","INT",4],["wal_level","TINYINT",1],["wal_fsync_period","INT",4],["wal_retention_period","INT",4],["wal_retention_size","BIGINT",8],["wal_roll_period","INT",4],["wal_segment_size","BIGINT",8]],"data":[["information_schema",null,null,16,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null],["performance_schema",null,null,10,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null]],"rows":2} +``` + +## 使用 dashboard 进行图形化管理 + + minikube 提供 dashboard 命令支持图形化管理界面。 + +``` +$ minikube dashboard +* Verifying dashboard health ... +* Launching proxy ... +* Verifying proxy health ... +* Opening http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ in your default browser... +http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ +``` + +对于某些公有云环境，minikube 绑定在 127.0.0.1 IP 地址上无法通过远程访问，需要使用 kubectl proxy 命令将端口映射到 0.0.0.0 IP 地址上，再通过浏览器访问虚拟机公网 IP 和端口以及相同的 dashboard URL 路径即可远程访问 dashboard。 + +``` +$ kubectl proxy --accept-hosts='^.*$' --address='0.0.0.0' ``` ## 集群扩容 @@ -252,14 +197,12 @@ TDengine 集群支持自动扩容： ```bash kubectl scale statefulsets tdengine --replicas=4 - ``` 上面命令行中参数 `--replica=4` 表示要将 TDengine 集群扩容到 4 个节点，执行后首先检查 POD 的状态： ```bash kubectl get pods -l app=tdengine - ``` 输出如下： @@ -270,102 +213,112 @@ tdengine-0 1/1 Running 0 161m tdengine-1 1/1 Running 0 161m tdengine-2 1/1 Running 0 32m tdengine-3 1/1 Running 0 32m - ``` 此时 POD 的状态仍然是 Running，TDengine 集群中的 dnode 状态要等 POD 状态为 `ready` 之后才能看到： ```bash -kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" - +kubectl exec -i -t tdengine-3 -- taos -s "show dnodes" ``` 扩容后的四节点 TDengine 集群的 dnode 列表: ``` -Welcome to the TDengine shell from Linux, Client Version:2.1.1.0 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:12.915 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:33.127 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 14:07:27.078 | | - 4 | tdengine-3.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 14:07:48.362 | | -Query OK, 4 row(s) in set (0.001293s) - + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | + 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | + 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | + 4 | tdengine-3.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:33:16.039 | | +Query OK, 4 rows in database (0.008377s) ``` ## 集群缩容 -TDengine 的缩容并没有自动化，我们尝试将一个三节点集群缩容到两节点。 +由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令，节点删除完成后再进行 Kubernetes 集群缩容。 -首先，确认一个三节点 TDengine 集群正常工作，在 TDengine CLI 中查看 dnode 的状态： +注意：由于 Kubernetes Statefulset 中 Pod 的只能按创建顺序逆序移除，所以 TDengine drop dnode 也需要按照创建顺序逆序移除，否则会导致 Pod 处于错误状态。 + +``` +$ kubectl exec -i -t tdengine-0 -- taos -s "drop dnode 4" +``` ```bash +$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" + +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. + taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 16:27:24.852 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:27:53.339 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:28:49.787 | | -Query OK, 3 row(s) in set (0.001101s) - + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | + 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | + 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | +Query OK, 3 rows in database (0.004861s) ``` -想要安全的缩容，首先需要将节点从 dnode 列表中移除，也即从集群中移除： +确认移除成功后（使用 kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" 查看和确认 dnode 列表），使用 kubectl 命令移除 POD： + +``` +kubectl scale statefulsets tdengine --replicas=3 +``` + +最后一个 POD 将会被删除。使用命令 kubectl get pods -l app=tdengine 查看POD状态： + +``` +$ kubectl get pods -l app=tdengine +NAME READY STATUS RESTARTS AGE +tdengine-0 1/1 Running 0 4m7s +tdengine-1 1/1 Running 0 3m55s +tdengine-2 1/1 Running 0 2m28s +``` + +POD删除后，需要手动删除PVC，否则下次扩容时会继续使用以前的数据导致无法正常加入集群。 ```bash -kubectl exec -i -t tdengine-0 -- taos -s "drop dnode 'tdengine-2.taosd.default.svc.cluster.local:6030'" - -``` - -通过 `show dondes` 命令确认移除成功后，移除相应的 POD： - -```bash -kubectl scale statefulsets tdengine --replicas=2 - -``` - -最后一个 POD 会被删除，使用 `kubectl get pods -l app=tdengine` 查看集群状态： - -``` -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 3h40m -tdengine-1 1/1 Running 0 3h40m - -``` - -POD 删除后，需要手动删除 PVC，否则下次扩容时会继续使用以前的数据导致无法正常加入集群。 - -```bash -kubectl delete pvc taosdata-tdengine-2 - +$ kubectl delete pvc taosdata-tdengine-3 ``` 此时的集群状态是安全的，需要时还可以再次进行扩容： ```bash -kubectl scale statefulsets tdengine --replicas=3 +$ kubectl scale statefulsets tdengine --replicas=4 +statefulset.apps/tdengine scaled +it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine +NAME READY STATUS RESTARTS AGE +tdengine-0 1/1 Running 0 35m +tdengine-1 1/1 Running 0 34m +tdengine-2 1/1 Running 0 12m +tdengine-3 0/1 ContainerCreating 0 4s +it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine +NAME READY STATUS RESTARTS AGE +tdengine-0 1/1 Running 0 35m +tdengine-1 1/1 Running 0 34m +tdengine-2 1/1 Running 0 12m +tdengine-3 0/1 Running 0 7s +it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" + +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. + +Server is Community Edition. - -``` - -`show dnodes` 输出如下： - -``` taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | +id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | ====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 16:27:24.852 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:27:53.339 | | - 4 | tdengine-2.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:40:49.177 | | - - +1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | +2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | +5 | tdengine-2.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:01:36.479 | | +6 | tdengine-3.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:13:54.411 | | +Query OK, 4 row(s) in set (0.001348s) ``` -## 删除集群 +## 清理 TDengine 集群 完整移除 TDengine 集群，需要分别清理 statefulset、svc、configmap、pvc。 @@ -381,26 +334,26 @@ kubectl delete configmap taoscfg ### 错误一 -扩容到四节点之后缩容到两节点，删除的 POD 会进入 offline 状态： +未进行 "drop dnode" 直接进行缩容，由于 TDengine 尚未删除节点，缩容 pod 导致 TDengine 集群中部分节点处于 offline 状态。 ``` -Welcome to the TDengine shell from Linux, Client Version:2.1.1.0 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. +$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" + +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. + +Server is Community Edition. taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | +id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | ====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:12.915 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:33.127 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 40 | offline | any | 2021-06-01 14:07:27.078 | status msg timeout | - 4 | tdengine-3.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 14:07:48.362 | status msg timeout | -Query OK, 4 row(s) in set (0.001236s) - - +1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | +2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | +5 | tdengine-2.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:01:36.479 | status msg timeout | +6 | tdengine-3.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:13:54.411 | status msg timeout | +Query OK, 4 row(s) in set (0.001323s) ``` -但 `drop dnode` 的行为按不会按照预期进行，且下次集群重启后，所有的 dnode 节点将无法启动 dropping 状态无法退出。 - ### 错误二 TDengine 集群会持有 replica 参数，如果缩容后的节点数小于这个值，集群将无法使用： From 99cabaee18cf41178b38108aa9c97d3bd1ec18dd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 14:07:51 +0800 Subject: [PATCH 110/165] fix(query):set correct offset. --- source/client/src/tmq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 8f866bde8e..24997a735c 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -2947,7 +2947,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; - char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols) - numOfCols * (sizeof(int8_t) + sizeof(int32_t)); + char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); int32_t* colLength = (int32_t*)pStart; pStart += sizeof(int32_t) * numOfCols; @@ -2977,6 +2977,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) char* data = pCol[k].pData + pCol[k].offset[j]; tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); } } else { From 249e2b12f66097d88882d8bcd19ccba6a9785216 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 10 Aug 2022 14:21:31 +0800 Subject: [PATCH 111/165] fix(query): fix interp + fill interpolation after last datablock --- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 +++ source/libs/executor/src/timewindowoperator.c | 35 ++++++++++--------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index cadb6fefd2..dd14aa6dc4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2631,6 +2631,11 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl STsdbReader* pPrevReader = pReader->innerReader[0]; SDataBlockIter* pBlockIter = &pPrevReader->status.blockIter; + code = tsdbTakeReadSnap(pPrevReader->pTsdb, &pPrevReader->pReadSnap); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + initFilesetIterator(&pPrevReader->status.fileIter, pPrevReader->pReadSnap->fs.aDFileSet, pPrevReader->order, pPrevReader->idStr); resetDataBlockIterator(&pPrevReader->status.blockIter, pPrevReader->order, pReader->status.pTableMap); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 51fea5ac24..0c62872299 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2115,7 +2115,7 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } -static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, +static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; @@ -2124,10 +2124,10 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp // output the result for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + //SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); switch (pSliceInfo->fillType) { @@ -2383,7 +2383,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2422,7 +2422,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2443,7 +2443,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2467,7 +2467,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepNextRows(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2501,7 +2501,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2532,15 +2532,16 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } - // check if need to interpolate after ts range - // except for fill(next), fill(linear) - while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (pResBlock->info.rows >= pResBlock->info.capacity) { - break; - } + } + + // check if need to interpolate after last datablock + // except for fill(next), fill(linear) + while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && pSliceInfo->fillType != TSDB_FILL_LINEAR) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; } } From 3807d00623785789375191dc2d51ad829a8a0d76 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 10 Aug 2022 14:26:03 +0800 Subject: [PATCH 112/165] test:update vnode case --- .../4dnode1mnode_basic_replica3_insertdatas_stop_leader.py | 2 +- tests/system-test/fulltest.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py index 76bb2cc996..1020139d63 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py @@ -270,7 +270,7 @@ class TDTestCase: if not vote_act: print("=======before_revote_leader_infos ======\n" , before_leader_infos) print("=======after_revote_leader_infos ======\n" , after_leader_infos) - tdLog.exit(" ===maybe revote not occured , there is no dnode offline ====") + tdLog.info(" ===maybe revote not occured , there is no dnode offline ====") else: for vgroup_info in vote_act: for ind , role in enumerate(vgroup_info): diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 7941e76df6..9cda2080d6 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -213,7 +213,7 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_query # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py -N 4 -M 1 +python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py -N 4 -M 1 From 465e79258afc01e8ed0dd52355cd2d564077c5d0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 10 Aug 2022 14:33:23 +0800 Subject: [PATCH 113/165] refact: a little optimize --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 73 ++++++++++++---------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 49ff8a732f..8ae0e824cf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -393,15 +393,16 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p SMemSkipListNode *px; SMemSkipListNode *pn; TSDBKEY *pTKey; - int c; - int backward = flags & SL_MOVE_BACKWARD; - int fromPos = flags & SL_MOVE_FROM_POS; + int32_t backward = flags & SL_MOVE_BACKWARD; + int32_t fromPos = flags & SL_MOVE_FROM_POS; if (backward) { px = pTbData->sl.pTail; - for (int8_t iLevel = pTbData->sl.maxLevel - 1; iLevel >= pTbData->sl.level; iLevel--) { - pos[iLevel] = px; + if (!fromPos) { + for (int8_t iLevel = pTbData->sl.level; iLevel < pTbData->sl.maxLevel; iLevel++) { + pos[iLevel] = px; + } } if (pTbData->sl.level) { @@ -412,7 +413,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p while (pn != pTbData->sl.pHead) { pTKey = (TSDBKEY *)SL_NODE_DATA(pn); - c = tsdbKeyCmprFn(pTKey, pKey); + int32_t c = tsdbKeyCmprFn(pTKey, pKey); if (c <= 0) { break; } else { @@ -427,8 +428,10 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p } else { px = pTbData->sl.pHead; - for (int8_t iLevel = pTbData->sl.maxLevel - 1; iLevel >= pTbData->sl.level; iLevel--) { - pos[iLevel] = px; + if (!fromPos) { + for (int8_t iLevel = pTbData->sl.level; iLevel < pTbData->sl.maxLevel; iLevel++) { + pos[iLevel] = px; + } } if (pTbData->sl.level) { @@ -437,9 +440,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { pn = SL_NODE_FORWARD(px, iLevel); while (pn != pTbData->sl.pTail) { - pTKey = (TSDBKEY *)SL_NODE_DATA(pn); - - c = tsdbKeyCmprFn(pTKey, pKey); + int32_t c = tsdbKeyCmprFn(SL_NODE_DATA(pn), pKey); if (c >= 0) { break; } else { @@ -480,36 +481,44 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN goto _exit; } pNode->level = level; - for (int8_t iLevel = 0; iLevel < level; iLevel++) { - SL_NODE_FORWARD(pNode, iLevel) = NULL; - SL_NODE_BACKWARD(pNode, iLevel) = NULL; - } - tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), pRow); - // put - for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) { - SMemSkipListNode *px = pos[iLevel]; + for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { + SMemSkipListNode *pn = pos[iLevel]; + SMemSkipListNode *px; if (forward) { - SMemSkipListNode *pNext = SL_NODE_FORWARD(px, iLevel); - - SL_NODE_FORWARD(pNode, iLevel) = pNext; - SL_NODE_BACKWARD(pNode, iLevel) = px; - - SL_NODE_BACKWARD(pNext, iLevel) = pNode; - SL_NODE_FORWARD(px, iLevel) = pNode; - } else { - SMemSkipListNode *pPrev = SL_NODE_BACKWARD(px, iLevel); + px = SL_NODE_FORWARD(pn, iLevel); + SL_NODE_BACKWARD(pNode, iLevel) = pn; SL_NODE_FORWARD(pNode, iLevel) = px; - SL_NODE_BACKWARD(pNode, iLevel) = pPrev; + } else { + px = SL_NODE_BACKWARD(pn, iLevel); - SL_NODE_FORWARD(pPrev, iLevel) = pNode; - SL_NODE_BACKWARD(px, iLevel) = pNode; + SL_NODE_BACKWARD(pNode, iLevel) = px; + SL_NODE_FORWARD(pNode, iLevel) = pn; } } + for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { + SMemSkipListNode *pn = pos[iLevel]; + SMemSkipListNode *px; + + if (forward) { + px = SL_NODE_FORWARD(pn, iLevel); + + SL_NODE_FORWARD(pn, iLevel) = pNode; + SL_NODE_BACKWARD(px, iLevel) = pNode; + } else { + px = SL_NODE_BACKWARD(pn, iLevel); + + SL_NODE_FORWARD(px, iLevel) = pNode; + SL_NODE_BACKWARD(pn, iLevel) = pNode; + } + + pos[iLevel] = pNode; + } + pTbData->sl.size++; if (pTbData->sl.level < pNode->level) { pTbData->sl.level = pNode->level; @@ -548,7 +557,7 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i // forward put rest data row.pTSRow = tGetSubmitBlkNext(&blkIter); if (row.pTSRow) { - for (int8_t iLevel = 0; iLevel < pTbData->sl.maxLevel; iLevel++) { + for (int8_t iLevel = pos[0]->level; iLevel < pTbData->sl.maxLevel; iLevel++) { pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); } do { From d5e3fb7341fb53c9fce83aad0c4d657fcdea7ed6 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 10 Aug 2022 15:02:12 +0800 Subject: [PATCH 114/165] doc: remove opentsdb to tdengine temporarily --- docs/zh/25-application/{03-immigrate.md => _03-immigrate.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/zh/25-application/{03-immigrate.md => _03-immigrate.md} (100%) diff --git a/docs/zh/25-application/03-immigrate.md b/docs/zh/25-application/_03-immigrate.md similarity index 100% rename from docs/zh/25-application/03-immigrate.md rename to docs/zh/25-application/_03-immigrate.md From 6d267810a2471a9edb22d3d51b3cba720353d5d0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 15:22:40 +0800 Subject: [PATCH 115/165] fix(query): fix the invalid length for json data. --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5c37e6dc15..acdb3b68b0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1747,7 +1747,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int pStart1 += colLen1; } - *(int32_t*)(pResultInfo->convertJson) = totalLen; + *(int32_t*)(pResultInfo->convertJson + 4) = totalLen; pResultInfo->pData = pResultInfo->convertJson; return TSDB_CODE_SUCCESS; } From 6dfa2836711a6a2a7b140955a79654c9402c11b9 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 10 Aug 2022 15:35:00 +0800 Subject: [PATCH 116/165] fix: use the pre agg value as double when generate block sma for float column and double column --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d612e9bb10..60f0b18a62 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1520,22 +1520,22 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { break; } case TSDB_DATA_TYPE_FLOAT: { - pColAgg->sum += colVal.value.f; - if (pColAgg->min > colVal.value.f) { - pColAgg->min = colVal.value.f; + *(double*)(&pColAgg->sum) += colVal.value.f; + if (*(double*)(&pColAgg->min) > colVal.value.f) { + *(double*)(&pColAgg->min) = colVal.value.f; } - if (pColAgg->max < colVal.value.f) { - pColAgg->max = colVal.value.f; + if (*(double*)(&pColAgg->max) < colVal.value.f) { + *(double*)(&pColAgg->max) = colVal.value.f; } break; } case TSDB_DATA_TYPE_DOUBLE: { - pColAgg->sum += colVal.value.d; - if (pColAgg->min > colVal.value.d) { - pColAgg->min = colVal.value.d; + *(double*)(&pColAgg->sum) += colVal.value.d; + if (*(double*)(&pColAgg->min) > colVal.value.d) { + *(double*)(&pColAgg->min) = colVal.value.d; } - if (pColAgg->max < colVal.value.d) { - pColAgg->max = colVal.value.d; + if (*(double*)(&pColAgg->max) < colVal.value.d) { + *(double*)(&pColAgg->max) = colVal.value.d; } break; } From 399d240df593b1b50ea7ad813823997c9f1d83fa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 10 Aug 2022 15:41:12 +0800 Subject: [PATCH 117/165] refactor index --- source/libs/index/src/index.c | 1 - source/libs/index/src/indexCache.c | 3 +-- source/libs/index/src/indexUtil.c | 3 ++- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index d61cd871ee..850ddd4970 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -485,7 +485,6 @@ static void idxMayMergeTempToFinalRslt(SArray* result, TFileValue* tfv, SIdxTRsl // handle last iterator idxTRsltMergeTo(tr, lv->tableId); } else { - // temp result saved in help tfileValueDestroy(tfv); } } else { diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 50745cd3fb..794b85d244 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -45,7 +45,7 @@ static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* ct, SIdxTRslt* tr, static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); static int32_t cacheSearchRange(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); -/*comm func of compare, used in (LE/LT/GE/GT compare)*/ +/*comm compare func, used in (LE/LT/GE/GT compare)*/ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s, RangeType type); static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); static int32_t cacheSearchEqual_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s); @@ -94,7 +94,6 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe if (0 == strcmp(c->colVal, pCt->colVal) && strlen(pCt->colVal) == strlen(c->colVal)) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid) - // taosArrayPush(result, &c->uid); *s = kTypeValue; } else if (c->operaType == DEL_VALUE) { INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid) diff --git a/source/libs/index/src/indexUtil.c b/source/libs/index/src/indexUtil.c index 1d20278895..3d083c1817 100644 --- a/source/libs/index/src/indexUtil.c +++ b/source/libs/index/src/indexUtil.c @@ -152,7 +152,7 @@ int verdataCompare(const void *a, const void *b) { int32_t cmp = compareUint64Val(&va->data, &vb->data); if (cmp == 0) { - cmp = 0 - compareUint32Val(&va->ver, &vb->data); + cmp = 0 - compareUint32Val(&va->ver, &vb->ver); return cmp; } return cmp; @@ -181,6 +181,7 @@ void idxTRsltDestroy(SIdxTRslt *tr) { taosArrayDestroy(tr->total); taosArrayDestroy(tr->add); taosArrayDestroy(tr->del); + taosMemoryFree(tr); } void idxTRsltMergeTo(SIdxTRslt *tr, SArray *result) { taosArraySort(tr->total, uidCompare); From 3327ae1556c14e02c1c2cb10672b09a6a3ad5b6f Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Wed, 10 Aug 2022 15:45:14 +0800 Subject: [PATCH 118/165] fix(taosAdapter): adapt to new raw_block --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 7a12413377..ee1e7d3b31 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 766dcc4 + GIT_TAG ed6a160 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 017a6c76662623f87231dad9436c7b29f1240a14 Mon Sep 17 00:00:00 2001 From: Bo Xiao Date: Wed, 10 Aug 2022 15:28:26 +0800 Subject: [PATCH 119/165] Update cpp.mdx --- docs/zh/14-reference/03-connector/cpp.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-connector/cpp.mdx b/docs/zh/14-reference/03-connector/cpp.mdx index aecf9fde12..3a8367ef33 100644 --- a/docs/zh/14-reference/03-connector/cpp.mdx +++ b/docs/zh/14-reference/03-connector/cpp.mdx @@ -279,7 +279,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 2. 调用 `taos_stmt_prepare()` 解析 INSERT 语句； 3. 如果 INSERT 语句中预留了表名但没有预留 TAGS，那么调用 `taos_stmt_set_tbname()` 来设置表名； 4. 如果 INSERT 语句中既预留了表名又预留了 TAGS（例如 INSERT 语句采取的是自动建表的方式），那么调用 `taos_stmt_set_tbname_tags()` 来设置表名和 TAGS 的值； -5. 调用 `taos_stmt_bind_param_batch()` 以多列的方式设置 VALUES 的值，或者调用 `taos_stmt_bind_param()` 以单行的方式设置 VALUES 的值； +5. 调用 `taos_stmt_bind_param_batch()` 以多行的方式设置 VALUES 的值，或者调用 `taos_stmt_bind_param()` 以单行的方式设置 VALUES 的值； 6. 调用 `taos_stmt_add_batch()` 把当前绑定的参数加入批处理； 7. 可以重复第 3 ～ 6 步，为批处理加入更多的数据行； 8. 调用 `taos_stmt_execute()` 执行已经准备好的批处理指令； From 18f6fb6123d8fb9386b821d04bd961cd252210bd Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 10 Aug 2022 16:04:03 +0800 Subject: [PATCH 120/165] enh(tmq): speed up consumer recover --- examples/c/tmq.c | 66 ++-- include/libs/stream/tstream.h | 2 +- include/util/taoserror.h | 1 + source/client/src/tmq.c | 173 +++-------- source/common/src/systable.c | 7 +- source/dnode/mnode/impl/src/mndConsumer.c | 12 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tq/tq.c | 3 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/stream/inc/streamInc.h | 2 +- source/libs/stream/src/streamData.c | 6 +- source/libs/stream/src/streamDispatch.c | 76 +---- source/libs/stream/src/streamExec.c | 34 +- source/libs/wal/src/walWrite.c | 2 + source/os/src/osSemaphore.c | 358 ++++++++++------------ source/util/src/terror.c | 1 + 16 files changed, 281 insertions(+), 466 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 1cdd4c02da..fc34915fe7 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -21,17 +21,17 @@ #include "taos.h" static int running = 1; -static char dbName[64] = "tmqdb"; -static char stbName[64] = "stb"; +static char dbName[64] = "tmqdb"; +static char stbName[64] = "stb"; static char topicName[64] = "topicname"; static int32_t msg_process(TAOS_RES* msg) { - char buf[1024]; + char buf[1024]; int32_t rows = 0; const char* topicName = tmq_get_topic_name(msg); - const char* dbName = tmq_get_db_name(msg); - int32_t vgroupId = tmq_get_vgroup_id(msg); + const char* dbName = tmq_get_db_name(msg); + int32_t vgroupId = tmq_get_vgroup_id(msg); printf("topic: %s\n", topicName); printf("db: %s\n", dbName); @@ -41,14 +41,14 @@ static int32_t msg_process(TAOS_RES* msg) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; - TAOS_FIELD* fields = taos_fetch_fields(msg); + TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); - int32_t* length = taos_fetch_lengths(msg); - int32_t precision = taos_result_precision(msg); - const char* tbName = tmq_get_table_name(msg); - rows++; + int32_t* length = taos_fetch_lengths(msg); + int32_t precision = taos_result_precision(msg); + const char* tbName = tmq_get_table_name(msg); + rows++; taos_print_row(buf, row, fields, numOfFields); - printf("row content from %s: %s\n", (tbName != NULL ? tbName : "null table"), buf); + printf("row content from %s: %s\n", (tbName != NULL ? tbName : "table null"), buf); } return rows; @@ -80,7 +80,8 @@ static int32_t init_env() { // create super table printf("create super table\n"); - pRes = taos_query(pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))"); + pRes = taos_query( + pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))"); if (taos_errno(pRes) != 0) { printf("failed to create super table stb, reason:%s\n", taos_errstr(pRes)); return -1; @@ -166,7 +167,6 @@ int32_t create_topic() { } taos_free_result(pRes); - // pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); pRes = taos_query(pConn, "create topic topicname as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topicname, reason:%s\n", taos_errstr(pRes)); @@ -184,26 +184,28 @@ void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { tmq_t* build_consumer() { tmq_conf_res_t code; - tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_t* conf = tmq_conf_new(); code = tmq_conf_set(conf, "enable.auto.commit", "true"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "group.id", "cgrpName"); if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "client.id", "user defined name"); + if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "td.connect.user", "root"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "td.connect.pass", "taosdata"); if (TMQ_CONF_OK != code) return NULL; - code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); + code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "experimental.snapshot.enable", "true"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "msg.with.table.name", "true"); if (TMQ_CONF_OK != code) return NULL; - - tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); - + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); tmq_conf_destroy(conf); return tmq; @@ -211,7 +213,7 @@ tmq_t* build_consumer() { tmq_list_t* build_topic_list() { tmq_list_t* topicList = tmq_list_new(); - int32_t code = tmq_list_append(topicList, "topicname"); + int32_t code = tmq_list_append(topicList, "topicname"); if (code) { return NULL; } @@ -228,18 +230,18 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topicList) { int32_t totalRows = 0; int32_t msgCnt = 0; - int32_t consumeDelay = 5000; + int32_t timeout = 5000; while (running) { - TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, consumeDelay); + TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, timeout); if (tmqmsg) { msgCnt++; totalRows += msg_process(tmqmsg); taos_free_result(tmqmsg); - } else { - break; - } + /*} else {*/ + /*break;*/ + } } - + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); } @@ -256,32 +258,30 @@ int main(int argc, char* argv[]) { tmq_t* tmq = build_consumer(); if (NULL == tmq) { - fprintf(stderr, "%% build_consumer() fail!\n"); + fprintf(stderr, "%% build_consumer() fail!\n"); return -1; } tmq_list_t* topic_list = build_topic_list(); if (NULL == topic_list) { return -1; - } - + } + basic_consume_loop(tmq, topic_list); code = tmq_unsubscribe(tmq); if (code) { fprintf(stderr, "%% Failed to unsubscribe: %s\n", tmq_err2str(code)); - } - else { + } else { fprintf(stderr, "%% unsubscribe\n"); } code = tmq_consumer_close(tmq); if (code) { fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); - } - else { + } else { fprintf(stderr, "%% Consumer closed\n"); } - + return 0; } diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9355d76dcb..32dc9e1866 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -118,7 +118,7 @@ typedef struct { int64_t sourceVer; int64_t reqId; - SArray* blocks; // SArray + SArray* blocks; // SArray } SStreamDataBlock; typedef struct { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 27fb057b44..145e2f7aad 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -617,6 +617,7 @@ int32_t* taosGetErrno(); //tmq #define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000) +#define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001) #ifdef __cplusplus } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 073f772ee4..c4375c85b4 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -122,6 +122,7 @@ enum { TMQ_CONSUMER_STATUS__INIT = 0, TMQ_CONSUMER_STATUS__READY, TMQ_CONSUMER_STATUS__NO_TOPIC, + TMQ_CONSUMER_STATUS__RECOVER, }; enum { @@ -134,10 +135,8 @@ typedef struct { // statistics int64_t pollCnt; // offset - /*int64_t committedOffset;*/ - /*int64_t currentOffset;*/ - STqOffsetVal committedOffsetNew; - STqOffsetVal currentOffsetNew; + STqOffsetVal committedOffset; + STqOffsetVal currentOffset; // connection info int32_t vgId; int32_t vgStatus; @@ -152,7 +151,6 @@ typedef struct { SArray* vgs; // SArray - int8_t isSchemaAdaptive; SSchemaWrapper schema; } SMqClientTopic; @@ -190,10 +188,9 @@ typedef struct { } SMqPollCbParam; typedef struct { - tmq_t* tmq; - int8_t automatic; - int8_t async; - /*int8_t freeOffsets;*/ + tmq_t* tmq; + int8_t automatic; + int8_t async; int32_t waitingRspNum; int32_t totalRspNum; int32_t rspErr; @@ -418,7 +415,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pOffset->val = pVg->currentOffsetNew; + pOffset->val = pVg->currentOffset; int32_t groupLen = strlen(tmq->groupId); memcpy(pOffset->subKey, tmq->groupId, groupLen); @@ -462,7 +459,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT pVg->vgId, pOffset->val.version); // TODO: put into cb - pVg->committedOffsetNew = pVg->currentOffsetNew; + pVg->committedOffset = pVg->currentOffset; pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; @@ -504,7 +501,6 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm pParamSet->tmq = tmq; pParamSet->automatic = 0; pParamSet->async = async; - /*pParamSet->freeOffsets = 1;*/ pParamSet->userCb = userCb; pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); @@ -518,7 +514,7 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); if (pVg->vgId != vgId) continue; - if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { goto FAIL; } @@ -550,8 +546,8 @@ FAIL: return 0; } -int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, - void* userParam) { +int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, + void* userParam) { int32_t code = -1; if (msg != NULL) { @@ -566,7 +562,6 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ pParamSet->tmq = tmq; pParamSet->automatic = automatic; pParamSet->async = async; - /*pParamSet->freeOffsets = 1;*/ pParamSet->userCb = userCb; pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); @@ -583,7 +578,9 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ tscDebug("consumer:%" PRId64 ", begin commit for topic %s, vgId:%d", tmq->consumerId, pTopic->topicName, pVg->vgId); - if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { + tscDebug("consumer: %ld, vg:%d, current %ld, committed %ld", tmq->consumerId, pVg->vgId, + pVg->currentOffset.version, pVg->committedOffset.version); if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { continue; } @@ -699,7 +696,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { tmqAskEp(tmq, true); taosTmrReset(tmqAssignAskEpTask, 1000, tmq, tmqMgmt.timer, &tmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { - tmqCommitInner2(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); + tmqCommitInner(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) { } else { @@ -888,12 +885,6 @@ FAIL: return NULL; } -#if 0 -int32_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { - return tmqCommitInner2(tmq, offsets, 0, async, tmq->commitCb, tmq->commitCbUserParam); -} -#endif - int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); @@ -967,7 +958,11 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = param.rspErr; if (code != 0) goto FAIL; + int32_t retryCnt = 0; while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + if (retryCnt++ > 10) { + goto FAIL; + } tscDebug("consumer not ready, retry"); taosMsleep(500); } @@ -1006,8 +1001,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t epoch = pParam->epoch; taosMemoryFree(pParam); if (code != 0) { - tscWarn("msg discard from vgId:%d, epoch %d, code:%x", vgId, epoch, code); + tscWarn("msg discard from vgId:%d, epoch %d, since %s", vgId, epoch, terrstr()); if (pMsg->pData) taosMemoryFree(pMsg->pData); + if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); + goto CREATE_MSG_FAIL; + } if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { @@ -1083,7 +1082,7 @@ CREATE_MSG_FAIL: return -1; } -bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { +bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); @@ -1112,10 +1111,10 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); char buf[80]; - tFormatOffset(buf, 80, &pVgCur->currentOffsetNew); + tFormatOffset(buf, 80, &pVgCur->currentOffset); tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); - taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffsetNew, sizeof(STqOffsetVal)); + taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(STqOffsetVal)); } } } @@ -1142,7 +1141,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { SMqClientVg clientVg = { .pollCnt = 0, - .currentOffsetNew = offsetNew, + .currentOffset = offsetNew, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, @@ -1166,93 +1165,6 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { return set; } -#if 0 -bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { - /*printf("call update ep %d\n", epoch);*/ - bool set = false; - int32_t topicNumGet = taosArrayGetSize(pRsp->topics); - char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; - tscDebug("consumer:%" PRId64 ", update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, - topicNumGet); - SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); - if (newTopics == NULL) { - return false; - } - SHashObj* pHash = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); - if (pHash == NULL) { - taosArrayDestroy(newTopics); - return false; - } - - // find topic, build hash - for (int32_t i = 0; i < topicNumGet; i++) { - SMqClientTopic topic = {0}; - SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); - topic.schema = pTopicEp->schema; - taosHashClear(pHash); - topic.topicName = strdup(pTopicEp->topic); - tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN); - - tscDebug("consumer:%" PRId64 ", update topic: %s", tmq->consumerId, topic.topicName); - int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); - for (int32_t j = 0; j < topicNumCur; j++) { - // find old topic - SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); - if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) { - int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); - tscDebug("consumer:%" PRId64 ", new vg num: %d", tmq->consumerId, vgNumCur); - if (vgNumCur == 0) break; - for (int32_t k = 0; k < vgNumCur; k++) { - SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); - sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); - tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey); - taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); - } - break; - } - } - - int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); - topic.vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); - for (int32_t j = 0; j < vgNumGet; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); - int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); - int64_t offset = pVgEp->offset; - tscDebug("consumer:%" PRId64 ", (epoch %d) original offset of vgId:%d is %" PRId64, tmq->consumerId, epoch, pVgEp->vgId, offset); - if (pOffset != NULL) { - offset = *pOffset; - tscDebug("consumer:%" PRId64 ", (epoch %d) receive offset of vgId:%d, full key is %s", tmq->consumerId, epoch, pVgEp->vgId, - vgKey); - } - tscDebug("consumer:%" PRId64 ", (epoch %d) offset of vgId:%d updated to %" PRId64, tmq->consumerId, epoch, pVgEp->vgId, offset); - SMqClientVg clientVg = { - .pollCnt = 0, - .currentOffset = offset, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet, - .vgStatus = TMQ_VG_STATUS__IDLE, - .vgSkipCnt = 0, - }; - taosArrayPush(topic.vgs, &clientVg); - set = true; - } - taosArrayPush(newTopics, &topic); - } - if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); - taosHashCleanup(pHash); - tmq->clientTopics = newTopics; - - if (taosArrayGetSize(tmq->clientTopics) == 0) - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC); - else - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); - - atomic_store_32(&tmq->epoch, epoch); - return set; -} -#endif - int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; @@ -1278,7 +1190,7 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); /*printf("rsp epoch %" PRId64 " sz %" PRId64 "\n", rsp.epoch, rsp.topics->size);*/ /*printf("tmq epoch %" PRId64 " sz %" PRId64 "\n", tmq->epoch, tmq->clientTopics->size);*/ - tmqUpdateEp2(tmq, head->epoch, &rsp); + tmqUpdateEp(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM); @@ -1430,7 +1342,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; /*pReq->currentOffset = reqOffset;*/ - pReq->reqOffset = pVg->currentOffsetNew; + pReq->reqOffset = pVg->currentOffset; pReq->reqId = generateRequestId(); pReq->useSnapshot = tmq->useSnapshot; @@ -1534,7 +1446,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { /*printf("send poll\n");*/ char offsetFormatBuf[80]; - tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffsetNew); + tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffset); tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64, tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); /*printf("send vgId:%d %" PRId64 "\n", pVg->vgId, pVg->currentOffset);*/ @@ -1552,7 +1464,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) { SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; - tmqUpdateEp2(tmq, rspWrapper->epoch, rspMsg); + tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg); /*tmqClearUnhandleMsg(tmq);*/ *pReset = true; } else { @@ -1586,7 +1498,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, * rspMsg->msg.rspOffset);*/ - pVg->currentOffsetNew = pollRspWrapper->dataRsp.rspOffset; + pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); if (pollRspWrapper->dataRsp.blockNum == 0) { taosFreeQitem(pollRspWrapper); @@ -1609,8 +1521,8 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, * rspMsg->msg.rspOffset);*/ - pVg->currentOffsetNew.version = pollRspWrapper->metaRsp.rspOffset; - pVg->currentOffsetNew.type = TMQ_OFFSET__LOG; + pVg->currentOffset.version = pollRspWrapper->metaRsp.rspOffset; + pVg->currentOffset.type = TMQ_OFFSET__LOG; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); @@ -1653,6 +1565,17 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { return NULL; } + if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { + int32_t retryCnt = 0; + while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + if (retryCnt++ > 10) { + return NULL; + } + tscDebug("consumer not ready, retry"); + taosMsleep(500); + } + } + while (1) { tmqHandleAllDelayedTask(tmq); if (tmqPollImpl(tmq, timeout) < 0) return NULL; @@ -3381,10 +3304,10 @@ int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { // - tmqCommitInner2(tmq, msg, 0, 1, cb, param); + tmqCommitInner(tmq, msg, 0, 1, cb, param); } int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) { // - return tmqCommitInner2(tmq, msg, 0, 0, NULL, NULL); + return tmqCommitInner(tmq, msg, 0, 0, NULL, NULL); } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 675d3bbfdd..edd718090f 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -16,8 +16,8 @@ #include "systable.h" #include "taos.h" #include "tdef.h" -#include "types.h" #include "tgrant.h" +#include "types.h" #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) @@ -243,8 +243,8 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, -// {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, -// {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, + // {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, + // {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema)}, {TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema)}, {TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, @@ -284,7 +284,6 @@ static const SSysDbTableSchema consumerSchema[] = { {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "topics", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 39b57f2966..3cdc53cd2c 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -131,8 +131,9 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId, mndConsumerStatusName(pConsumer->status)); - if (pConsumer->status != MQ_CONSUMER_STATUS__READY) { + if (pConsumer->status != MQ_CONSUMER_STATUS__LOST_REBD) { mndReleaseConsumer(pMnode, pConsumer); + terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } @@ -275,6 +276,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int32_t status = atomic_load_32(&pConsumer->status); if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + mInfo("try to recover consumer %ld", consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -305,15 +307,14 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); -#if 1 atomic_store_32(&pConsumer->hbStatus, 0); -#endif // 1. check consumer status int32_t status = atomic_load_32(&pConsumer->status); #if 1 if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + mInfo("try to recover consumer %ld", consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -326,6 +327,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { #endif if (status != MQ_CONSUMER_STATUS__READY) { + mInfo("consumer %ld not ready, status: %s", consumerId, mndConsumerStatusName(status)); terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } @@ -939,10 +941,6 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * colDataAppend(pColInfo, numOfRows, NULL, true); } - // pid - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->pid, true); - // end point pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 4c6320ecb5..bce2b642e8 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -157,7 +157,7 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); int32_t tqProcessCheckAlterInfoReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); -int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen, int64_t ver); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 02ca248054..112543e340 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -183,7 +183,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con return 0; } -int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen, int64_t ver) { STqOffset offset = {0}; SDecoder decoder; tDecoderInit(&decoder, msg, msgLen); @@ -302,6 +302,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqError("tmq poll: consumer handle mismatch for consumer:%" PRId64 ", in vgId:%d, subkey %s, handle consumer id %" PRId64, consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0cb66484c0..2d181c5458 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -204,7 +204,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp break; case TDMT_VND_MQ_COMMIT_OFFSET: if (tqProcessOffsetCommitReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), - pMsg->contLen - sizeof(SMsgHead)) < 0) { + pMsg->contLen - sizeof(SMsgHead), version) < 0) { goto _err; } break; diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 7dc8d822e9..3776cb261f 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -44,7 +44,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); -SStreamQueueItem* streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); void streamFreeQitem(SStreamQueueItem* data); #ifdef __cplusplus diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 54014d7df9..c584c2a15f 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -125,7 +125,7 @@ void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) { } } -SStreamQueueItem* streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { ASSERT(elem); if (dst->type == STREAM_INPUT__DATA_BLOCK && elem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; @@ -172,8 +172,8 @@ void streamFreeQitem(SStreamQueueItem* data) { int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { - void* data = taosArrayGetP(pMerge->reqs, i); - taosMemoryFree(data); + void* dataStr = taosArrayGetP(pMerge->reqs, i); + taosMemoryFree(dataStr); taosMemoryFree(pRef); } } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9d98afa65a..c78ff0756f 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -370,80 +370,6 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat return 0; } -int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { - void* buf = NULL; - int32_t code = -1; - int32_t blockNum = taosArrayGetSize(data->blocks); - ASSERT(blockNum != 0); - - SStreamDispatchReq req = { - .streamId = pTask->streamId, - .dataSrcVgId = data->srcVgId, - .upstreamTaskId = pTask->taskId, - .upstreamChildId = pTask->selfChildId, - .upstreamNodeId = pTask->nodeId, - .blockNum = blockNum, - }; - - req.data = taosArrayInit(blockNum, sizeof(void*)); - req.dataLen = taosArrayInit(blockNum, sizeof(int32_t)); - if (req.data == NULL || req.dataLen == NULL) { - goto FAIL; - } - for (int32_t i = 0; i < blockNum; i++) { - SSDataBlock* pDataBlock = taosArrayGet(data->blocks, i); - if (streamAddBlockToDispatchMsg(pDataBlock, &req) < 0) { - goto FAIL; - } - } - int32_t vgId = 0; - int32_t downstreamTaskId = 0; - // find ep - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - vgId = pTask->fixedEpDispatcher.nodeId; - *ppEpSet = &pTask->fixedEpDispatcher.epSet; - downstreamTaskId = pTask->fixedEpDispatcher.taskId; - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - // TODO get ctbName for each block - SSDataBlock* pBlock = taosArrayGet(data->blocks, 0); - char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId); - // TODO: get hash function by hashMethod - - // get groupId, compute hash value - uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); - - // get node - // TODO: optimize search process - SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t sz = taosArrayGetSize(vgInfo); - for (int32_t i = 0; i < sz; i++) { - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - ASSERT(pVgInfo->vgId > 0); - if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { - vgId = pVgInfo->vgId; - downstreamTaskId = pVgInfo->taskId; - *ppEpSet = &pVgInfo->epSet; - break; - } - } - } - - ASSERT(vgId > 0 || vgId == SNODE_HANDLE); - req.taskId = downstreamTaskId; - - qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, - downstreamTaskId, vgId); - - streamDispatchOneReq(pTask, &req, vgId, *ppEpSet); - - code = 0; -FAIL: - if (code < 0 && buf) rpcFreeCont(buf); - if (req.data) taosArrayDestroyP(req.data, (FDelete)taosMemoryFree); - if (req.dataLen) taosArrayDestroy(req.dataLen); - return code; -} - int32_t streamDispatch(SStreamTask* pTask) { ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); @@ -461,7 +387,7 @@ int32_t streamDispatch(SStreamTask* pTask) { } ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - qDebug("stream continue dispatching: task %d", pTask->taskId); + qDebug("stream dispatching: task %d", pTask->taskId); int32_t code = 0; if (streamDispatchAllBlocks(pTask, pBlock) < 0) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 72249c5181..06ca26f029 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -82,17 +82,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* return 0; } -#if 0 -static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock* pBlock) { - ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - int32_t childId = pBlock->childId; - int64_t ver = pBlock->sourceVer; - SStreamChildEpInfo* pChildInfo = taosArrayGetP(pTask->childEpInfo, childId); - /*pChildInfo-> = ver;*/ - return 0; -} -#endif - int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch) { ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); @@ -150,10 +139,11 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch) return 0; } + // TODO: handle version int32_t streamExecForAll(SStreamTask* pTask) { while (1) { - int32_t cnt = 1; + int32_t batchCnt = 1; void* data = NULL; while (1) { SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); @@ -169,13 +159,12 @@ int32_t streamExecForAll(SStreamTask* pTask) { } } else { void* newRet; - if ((newRet = streamAppendQueueItem(data, qItem)) == NULL) { + if ((newRet = streamMergeQueueItem(data, qItem)) == NULL) { streamQueueProcessFail(pTask->inputQueue); break; } else { - cnt++; + batchCnt++; data = newRet; - /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ streamQueueProcessSuccess(pTask->inputQueue); } } @@ -198,16 +187,14 @@ int32_t streamExecForAll(SStreamTask* pTask) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); + qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, batchCnt); streamTaskExecImpl(pTask, data, pRes); qDebug("stream task %d exec end", pTask->taskId); if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); if (qRes == NULL) { - // TODO log failed ver - streamQueueProcessFail(pTask->inputQueue); - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); streamFreeQitem(data); return -1; } @@ -218,17 +205,18 @@ int32_t streamExecForAll(SStreamTask* pTask) { SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; + } else if (((SStreamQueueItem*)data)->type == STREAM_INPUT__MERGED_SUBMIT) { + SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)data; + qRes->childId = pTask->selfChildId; + qRes->sourceVer = pMerged->ver; } if (streamTaskOutput(pTask, qRes) < 0) { - // TODO save failed ver - /*streamQueueProcessFail(pTask->inputQueue);*/ taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - taosFreeQitem(qRes); streamFreeQitem(data); + taosFreeQitem(qRes); return -1; } - /*streamQueueProcessSuccess(pTask->inputQueue);*/ } else { taosArrayDestroy(pRes); } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index ad571a9e82..fad3977a21 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -421,6 +421,8 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); + wDebug("vgId:%d, wal write log %ld, msgType: %s", pWal->cfg.vgId, index, TMSG_INFO(msgType)); + if (taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 63c414386b..ab4fb551fe 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -76,8 +76,8 @@ int32_t tsem_wait(tsem_t* sem) { int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { struct timespec ts, rel; - FILETIME ft_before, ft_after; - int rc; + FILETIME ft_before, ft_after; + int rc; rel.tv_sec = 0; rel.tv_nsec = nanosecs; @@ -217,7 +217,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int e = errno; // if (e == EEXIST) continue; // if (e == EINTR) continue; -// fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, +// fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem, // e, strerror(e)); // abort(); // } while (p->sem == SEM_FAILED); @@ -231,7 +232,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value); // if (ret != KERN_SUCCESS) { -// fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // // we fail-fast here, because we have less-doc about semaphore_create for the moment // abort(); @@ -258,8 +260,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // struct tsem_s *p = *sem; // if (!p->valid) { -// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); -// abort(); +// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); abort(); // } // #ifdef SEM_USE_PTHREAD // if (taosThreadMutexLock(&p->lock)) { @@ -270,7 +272,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // p->val -= 1; // if (p->val < 0) { // if (taosThreadCondWait(&p->cond, &p->lock)) { -// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // abort(); // } @@ -297,8 +300,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // struct tsem_s *p = *sem; // if (!p->valid) { -// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); -// abort(); +// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); abort(); // } // #ifdef SEM_USE_PTHREAD // if (taosThreadMutexLock(&p->lock)) { @@ -309,7 +312,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // p->val += 1; // if (p->val <= 0) { // if (taosThreadCondSignal(&p->cond)) { -// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // abort(); // } @@ -332,7 +336,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int tsem_destroy(tsem_t *sem) { // // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); // if (!*sem) { -// // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); +// // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); // // abort(); // return 0; // } @@ -370,7 +375,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int r = sem_unlink(name); // if (r) { // int e = errno; -// fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, +// fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem, // e, strerror(e)); // abort(); // } @@ -385,225 +391,189 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // *sem = NULL; // return 0; // } -typedef struct -{ - pthread_mutex_t count_lock; - pthread_cond_t count_bump; - unsigned int count; -}bosal_sem_t; +typedef struct { + pthread_mutex_t count_lock; + pthread_cond_t count_bump; + unsigned int count; +} bosal_sem_t; -int tsem_init(tsem_t *psem, int flags, unsigned int count) -{ - bosal_sem_t *pnewsem; - int result; +int tsem_init(tsem_t *psem, int flags, unsigned int count) { + bosal_sem_t *pnewsem; + int result; - pnewsem = (bosal_sem_t *)malloc(sizeof(bosal_sem_t)); - if (! pnewsem) - { - return -1; - } - result = pthread_mutex_init(&pnewsem->count_lock, NULL); - if (result) - { - free(pnewsem); - return result; - } - result = pthread_cond_init(&pnewsem->count_bump, NULL); - if (result) - { - pthread_mutex_destroy(&pnewsem->count_lock); - free(pnewsem); - return result; - } - pnewsem->count = count; - *psem = (tsem_t)pnewsem; - return 0; + pnewsem = (bosal_sem_t *)malloc(sizeof(bosal_sem_t)); + if (!pnewsem) { + return -1; + } + result = pthread_mutex_init(&pnewsem->count_lock, NULL); + if (result) { + free(pnewsem); + return result; + } + result = pthread_cond_init(&pnewsem->count_bump, NULL); + if (result) { + pthread_mutex_destroy(&pnewsem->count_lock); + free(pnewsem); + return result; + } + pnewsem->count = count; + *psem = (tsem_t)pnewsem; + return 0; } -int tsem_destroy(tsem_t *psem) -{ - bosal_sem_t *poldsem; +int tsem_destroy(tsem_t *psem) { + bosal_sem_t *poldsem; - if (! psem) - { - return EINVAL; - } - poldsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + poldsem = (bosal_sem_t *)*psem; - pthread_mutex_destroy(&poldsem->count_lock); - pthread_cond_destroy(&poldsem->count_bump); - free(poldsem); - return 0; + pthread_mutex_destroy(&poldsem->count_lock); + pthread_cond_destroy(&poldsem->count_bump); + free(poldsem); + return 0; } -int tsem_post(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_post(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - pxsem->count = pxsem->count + 1; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + pxsem->count = pxsem->count + 1; - xresult = pthread_cond_signal(&pxsem->count_bump); + xresult = pthread_cond_signal(&pxsem->count_bump); - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_trywait(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_trywait(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count > 0) - { - pxsem->count--; - } - else - { - xresult = EAGAIN; - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + if (pxsem->count > 0) { + pxsem->count--; + } else { + xresult = EAGAIN; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_wait(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_wait(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count == 0) - { - xresult = pthread_cond_wait(&pxsem->count_bump, &pxsem->count_lock); + if (pxsem->count == 0) { + xresult = pthread_cond_wait(&pxsem->count_bump, &pxsem->count_lock); + } + if (!xresult) { + if (pxsem->count > 0) { + pxsem->count--; } - if (! xresult) - { - if (pxsem->count > 0) - { - pxsem->count--; - } - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_timewait(tsem_t *psem, int64_t nanosecs) -{ +int tsem_timewait(tsem_t *psem, int64_t nanosecs) { struct timespec abstim = { .tv_sec = 0, .tv_nsec = nanosecs, }; - bosal_sem_t *pxsem; - int result, xresult; + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count == 0) - { - xresult = pthread_cond_timedwait(&pxsem->count_bump, &pxsem->count_lock, &abstim); + if (pxsem->count == 0) { + xresult = pthread_cond_timedwait(&pxsem->count_bump, &pxsem->count_lock, &abstim); + } + if (!xresult) { + if (pxsem->count > 0) { + pxsem->count--; } - if (! xresult) - { - if (pxsem->count > 0) - { - pxsem->count--; - } - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -bool taosCheckPthreadValid(TdThread thread) { +bool taosCheckPthreadValid(TdThread thread) { int32_t ret = taosThreadKill(thread, 0); if (ret == ESRCH) return false; if (ret == EINVAL) return false; // alive return true; - } +} int64_t taosGetSelfPthreadId() { TdThread thread = taosThreadSelf(); @@ -650,7 +620,13 @@ int64_t taosGetSelfPthreadId() { int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; } void taosResetPthread(TdThread* thread) { *thread = 0; } bool taosComparePthread(TdThread first, TdThread second) { return first == second; } -int32_t taosGetPId() { return getpid(); } + +int32_t taosGetPId() { + static __thread int32_t pid = 0; + if (pid != 0) return pid; + pid = getpid(); + return pid; +} int32_t taosGetAppName(char* name, int32_t* len) { const char* self = "/proc/self/exe"; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4780d85a30..93b91739a3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -621,6 +621,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Invalid index file" //tmq TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") #ifdef TAOS_ERROR_C }; From c8ffd4f1610d6099e569d9c73511b2cd045708b4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 10 Aug 2022 16:12:39 +0800 Subject: [PATCH 121/165] docs: update helm doc for 3.0 (#15938) --- docs/zh/10-deployment/05-helm.md | 47 ++++++++++---------------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/docs/zh/10-deployment/05-helm.md b/docs/zh/10-deployment/05-helm.md index 0bbd986b4b..9a723ff62f 100644 --- a/docs/zh/10-deployment/05-helm.md +++ b/docs/zh/10-deployment/05-helm.md @@ -22,7 +22,7 @@ Helm 会使用 kubectl 和 kubeconfig 的配置来操作 Kubernetes，可以参 TDengine Chart 尚未发布到 Helm 仓库，当前可以从 GitHub 直接下载： ```bash -wget https://github.com/taosdata/TDengine-Operator/raw/main/helm/tdengine-0.3.0.tgz +wget https://github.com/taosdata/TDengine-Operator/raw/3.0/helm/tdengine-3.0.0.tgz ``` @@ -38,7 +38,7 @@ kubectl get storageclass 之后，使用 helm 命令安装： ```bash -helm install tdengine tdengine-0.3.0.tgz \ +helm install tdengine tdengine-3.0.0.tgz \ --set storage.className= ``` @@ -46,7 +46,7 @@ helm install tdengine tdengine-0.3.0.tgz \ 在 minikube 环境下，可以设置一个较小的容量避免超出磁盘可用空间： ```bash -helm install tdengine tdengine-0.3.0.tgz \ +helm install tdengine tdengine-3.0.0.tgz \ --set storage.className=standard \ --set storage.dataSize=2Gi \ --set storage.logSize=10Mi @@ -83,14 +83,14 @@ TDengine 支持 `values.yaml` 自定义。 通过 helm show values 可以获取 TDengine Chart 支持的全部 values 列表： ```bash -helm show values tdengine-0.3.0.tgz +helm show values tdengine-3.0.0.tgz ``` 你可以将结果保存为 values.yaml，之后可以修改其中的各项参数，如 replica 数量，存储类名称，容量大小，TDengine 配置等，然后使用如下命令安装 TDengine 集群： ```bash -helm install tdengine tdengine-0.3.0.tgz -f values.yaml +helm install tdengine tdengine-3.0.0.tgz -f values.yaml ``` @@ -107,37 +107,17 @@ image: prefix: tdengine/tdengine #pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - #tag: "2.4.0.5" +# tag: "3.0.0.0" service: # ClusterIP is the default service type, use NodeIP only if you know what you are doing. type: ClusterIP ports: # TCP range required - tcp: - [ - 6030, - 6031, - 6032, - 6033, - 6034, - 6035, - 6036, - 6037, - 6038, - 6039, - 6040, - 6041, - 6042, - 6043, - 6044, - 6045, - 6060, - ] - # UDP range 6030-6039 - udp: [6030, 6031, 6032, 6033, 6034, 6035, 6036, 6037, 6038, 6039] + tcp: [6030, 6041, 6042, 6043, 6044, 6046, 6047, 6048, 6049, 6060] + # UDP range + udp: [6044, 6045] -arbitrator: true # Set timezone here, not in taoscfg timezone: "Asia/Shanghai" @@ -182,11 +162,14 @@ clusterDomainSuffix: "" # # Btw, keep quotes "" around the value like below, even the value will be number or not. taoscfg: + # Starts as cluster or not, must be 0 or 1. + # 0: all pods will start as a seperate TDengine server + # 1: pods will start as TDengine server cluster. [default] + CLUSTER: "1" + # number of replications, for cluster only TAOS_REPLICA: "1" - # number of management nodes in the system - TAOS_NUM_OF_MNODES: "1" # number of days per DB file # TAOS_DAYS: "10" @@ -422,7 +405,7 @@ kubectl --namespace default exec $POD_NAME -- taos -s 'drop dnode " Date: Wed, 10 Aug 2022 16:20:44 +0800 Subject: [PATCH 122/165] fix: return init agg failure when create operators to avoid coredump --- source/libs/executor/src/executorimpl.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7e77799702..47d0ed3131 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3432,7 +3432,11 @@ int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInf return code; } - doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey); + code = doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + for (int32_t i = 0; i < numOfCols; ++i) { pSup->pCtx[i].pBuf = pAggSup->pResultBuf; } From 788735f7e693d508c262cf2bb827303764432b21 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 10 Aug 2022 16:25:00 +0800 Subject: [PATCH 123/165] fix: add qerror log when create disk based result buf failed --- source/libs/executor/src/executorimpl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 47d0ed3131..bd56bc2745 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3413,6 +3413,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n } int32_t code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir); if (code != TSDB_CODE_SUCCESS) { + qError("Create agg result buf failed since %s", tstrerror(code)); return code; } From 6fc38165fb7d009d04878eec58be29a2958fd64b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 10 Aug 2022 16:29:55 +0800 Subject: [PATCH 124/165] fix(query): fix linear interpolation issue between two blocks TD-18309 --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/inc/tfill.h | 1 - source/libs/executor/src/timewindowoperator.c | 68 ++++++++++++++----- 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8439cf700d..75d7c1c1ff 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -739,6 +739,7 @@ typedef struct STimeSliceOperatorInfo { SArray* pPrevRow; // SArray SArray* pNextRow; // SArray SArray* pLinearInfo; // SArray + bool fillLastPoint; bool isPrevRowSet; bool isNextRowSet; int32_t fillType; // fill type diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index c2de48d0eb..1e95f4a04f 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -37,7 +37,6 @@ typedef struct SFillLinearInfo { SPoint start; SPoint end; bool hasNull; - bool fillLastPoint; int16_t type; int32_t bytes; } SFillLinearInfo; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0c62872299..c7fa444cc6 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2087,8 +2087,10 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock pSliceInfo->isNextRowSet = true; } -static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { +static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex, + bool isLastRow) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + bool fillLastPoint = pSliceInfo->fillLastPoint; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); @@ -2096,16 +2098,22 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo // null data should not be kept since it can not be used to perform interpolation if (!colDataIsNull_s(pColInfoData, i)) { - int64_t startKey = *(int64_t*)colDataGetData(pTsCol, rowIndex); - int64_t endKey = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1); - pLinearInfo->start.key = startKey; - pLinearInfo->end.key = endKey; + if (isLastRow) { + pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + } else if (fillLastPoint) { + pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + } else { + pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1); - char* val; - val = colDataGetData(pColInfoData, rowIndex); - memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes); - val = colDataGetData(pColInfoData, rowIndex + 1); - memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes); + char* val; + val = colDataGetData(pColInfoData, rowIndex); + memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes); + val = colDataGetData(pColInfoData, rowIndex + 1); + memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes); + } pLinearInfo->hasNull = false; } else { @@ -2113,6 +2121,8 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } } + pSliceInfo->fillLastPoint = isLastRow ? true : false; + } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, @@ -2269,7 +2279,7 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB } pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo)); - if (pInfo->pNextRow == NULL) { + if (pInfo->pLinearInfo == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -2283,15 +2293,20 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes); linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes); linearInfo.hasNull = false; - linearInfo.fillLastPoint = false; linearInfo.type = pColInfo->info.type; linearInfo.bytes = pColInfo->info.bytes; taosArrayPush(pInfo->pLinearInfo, &linearInfo); } + pInfo->fillLastPoint = false; + return TSDB_CODE_SUCCESS; } +static bool needToFillLastPoint(STimeSliceOperatorInfo* pSliceInfo) { + return (pSliceInfo->fillLastPoint == true && pSliceInfo->fillType == TSDB_FILL_LINEAR); +} + static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { int32_t code; code = initPrevRowsKeeper(pInfo, pBlock); @@ -2356,6 +2371,23 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { for (int32_t i = 0; i < pBlock->info.rows; ++i) { int64_t ts = *(int64_t*)colDataGetData(pTsCol, i); + if (i == 0 && needToFillLastPoint(pSliceInfo)) { // first row in current block + doKeepLinearInfo(pSliceInfo, pBlock, i, false); + while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } + + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; + } + } + if (ts == pSliceInfo->current) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; @@ -2374,9 +2406,10 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // for linear interpolation, always fill value between this and next points; // if its the first point in data block, also fill values between previous(if there's any) and this point; - // if its the last point in data block, no need to fill, but reserve this point as the start value for next data block. + // if its the last point in data block, no need to fill, but reserve this point as the start value and do + // the interpolation when processing next data block. if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - doKeepLinearInfo(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (i < pBlock->info.rows - 1) { @@ -2396,6 +2429,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } + } else {// it is the last row of current block + //store ts value as start, and calculate interp value when processing next block + doKeepLinearInfo(pSliceInfo, pBlock, i, true); } } else { // non-linear interpolation pSliceInfo->current = @@ -2414,7 +2450,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepPrevRows(pSliceInfo, pBlock, i); if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - doKeepLinearInfo(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i, false); // no need to increate pSliceInfo->current here //pSliceInfo->current = // taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -2494,7 +2530,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - doKeepLinearInfo(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (i < pBlock->info.rows - 1) { From 5f7b7a55df5d3511c9bc563506a689fad79b52a6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 10 Aug 2022 16:33:12 +0800 Subject: [PATCH 125/165] fix: fix memory leaks --- source/libs/executor/src/joinoperator.c | 2 ++ source/libs/executor/src/scanoperator.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 2f9eff50b6..7d2b84d0f0 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -132,6 +132,8 @@ void destroyMergeJoinOperator(void* param, int32_t numOfOutput) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; nodesDestroyNode(pJoinOperator->pCondAfterMerge); + pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); + taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b21e937eb9..8e9ae3cd15 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2570,6 +2570,8 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { STagScanInfo* pInfo = (STagScanInfo*)param; pInfo->pRes = blockDataDestroy(pInfo->pRes); + taosArrayDestroy(pInfo->pColMatchInfo); + taosMemoryFreeClear(param); } From 95f696b30db2a7a082c8f29adeeba2dbaede205b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 10 Aug 2022 17:23:07 +0800 Subject: [PATCH 126/165] enh(tmq): prevent drop db when topic exists --- examples/rust | 1 - include/util/taoserror.h | 2 ++ source/client/src/tmq.c | 21 +++++---------------- source/dnode/mnode/impl/inc/mndTopic.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 10 ++++++---- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndTopic.c | 20 ++++++++++++++++++++ source/util/src/terror.c | 1 + tools/taos-tools | 1 - 9 files changed, 36 insertions(+), 23 deletions(-) delete mode 160000 examples/rust delete mode 160000 tools/taos-tools diff --git a/examples/rust b/examples/rust deleted file mode 160000 index 7ed7a97715..0000000000 --- a/examples/rust +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7ed7a97715388fa144718764d6bf20f9bfc29a12 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ce73dceec2..3ca6978156 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -284,11 +284,13 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_CONSUMER_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_TOPIC_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x03EB) #define TSDB_CODE_MND_CGROUP_USED TAOS_DEF_ERROR_CODE(0, 0x03EC) +#define TSDB_CODE_MND_TOPIC_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03ED) // mnode-stream #define TSDB_CODE_MND_STREAM_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F0) #define TSDB_CODE_MND_STREAM_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F1) #define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2) +#define TSDB_CODE_MND_STREAM_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03F3) // mnode-sma #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index c4375c85b4..c0874caf95 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -204,7 +204,7 @@ typedef struct { typedef struct { SMqCommitCbParamSet* params; STqOffset* pOffset; -} SMqCommitCbParam2; +} SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); @@ -368,8 +368,8 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { return sprintf(dst, "%s:%d", topicName, vg); } -int32_t tmqCommitCb2(void* param, SDataBuf* pBuf, int32_t code) { - SMqCommitCbParam2* pParam = (SMqCommitCbParam2*)param; +int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { + SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; // push into array #if 0 @@ -440,7 +440,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT tEncodeSTqOffset(&encoder, pOffset); // build param - SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); + SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); pParam->params = pParamSet; pParam->pOffset = pOffset; @@ -465,7 +465,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT pMsgSendInfo->requestObjRefId = 0; pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; - pMsgSendInfo->fp = tmqCommitCb2; + pMsgSendInfo->fp = tmqCommitCb; pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; // send msg @@ -1313,17 +1313,6 @@ int32_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { #endif SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { - /*int64_t reqOffset;*/ - /*if (pVg->currentOffset >= 0) {*/ - /*reqOffset = pVg->currentOffset;*/ - /*} else {*/ - /*if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {*/ - /*tscError("unable to poll since no committed offset but reset offset is set to none");*/ - /*return NULL;*/ - /*}*/ - /*reqOffset = tmq->resetOffsetCfg;*/ - /*}*/ - SMqPollReq* pReq = taosMemoryCalloc(1, sizeof(SMqPollReq)); if (pReq == NULL) { return NULL; diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index c5c4800e02..9f516904ce 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -32,6 +32,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic); SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw); int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); +int32_t mndCheckTopicExist(SMnode *pMnode, SDbObj *pDb); const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 89ea7abc23..9dde083f50 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -995,11 +995,13 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); mndTransSetDbName(pTrans, pDb->name, NULL); + if (mndCheckTopicExist(pMnode, pDb) < 0) goto _OVER; + if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER; - if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER; - if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER; - if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER; + /*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ + /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ + /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; @@ -1706,7 +1708,7 @@ static void setPerfSchemaDbCfg(SDbObj *pDbObj) { static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; int32_t *numOfTables = p1; - int64_t uid = *(int64_t*)p2; + int64_t uid = *(int64_t *)p2; if (pVgroup->dbUid == uid) { *numOfTables += pVgroup->numOfTables; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7bde05aca5..0f20188033 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -837,7 +837,7 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { sdbCancelFetch(pSdb, pIter); mError("db:%s, failed to drop stream:%s since sourceDbUid:%" PRId64 " not match with targetDbUid:%" PRId64, pDb->name, pStream->name, pStream->sourceDbUid, pStream->targetDbUid); - terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; + terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; return -1; } else { #if 0 diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index f343c2e055..820bb4b636 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -782,6 +782,26 @@ static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } +int32_t mndCheckTopicExist(SMnode *pMnode, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + SMqTopicObj *pTopic = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); + if (pIter == NULL) break; + + if (pTopic->dbUid == pDb->uid) { + sdbRelease(pSdb, pTopic); + terrno = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED; + return -1; + } + + sdbRelease(pSdb, pTopic); + } + return 0; +} + int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t code = 0; SSdb *pSdb = pMnode->pSdb; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index f0a58c9bdc..bb26f9b2f7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -286,6 +286,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST, "Subcribe not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_OFFSET_NOT_EXIST, "Offset not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_SUBSCRIBED, "Topic subscribed cannot be dropped") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_MUST_BE_DELETED, "Topic must be dropped first") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being used by some consumer") // mnode-stream diff --git a/tools/taos-tools b/tools/taos-tools deleted file mode 160000 index 3c7dafeea3..0000000000 --- a/tools/taos-tools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3c7dafeea3e558968165b73bee0f51024898e3da From 1d6d0aa2db2ffc8024f27a5145b6811f4469f879 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 10 Aug 2022 17:26:44 +0800 Subject: [PATCH 127/165] enh: row iter and fetch optimization --- include/common/tdataformat.h | 2 - include/common/trow.h | 13 +- source/common/src/trow.c | 189 +++++++------------------ source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/libs/executor/src/tsimplehash.c | 79 +++++------ source/libs/parser/src/parInsert.c | 4 +- 7 files changed, 108 insertions(+), 183 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 311eb72b4d..3679b3773b 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -200,8 +200,6 @@ struct STag { #if 1 //================================================================================================================================================ // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP -#define TD_SUPPORT_READ2 -#define TD_SUPPORT_BACK2 // suppport back compatibility of 2.0 #define TASSERT(x) ASSERT(x) diff --git a/include/common/trow.h b/include/common/trow.h index bd8510dde6..a1db2ef708 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -195,6 +195,12 @@ static FORCE_INLINE void tdRowSetVal(SCellVal *pVal, uint8_t valType, void *val) pVal->valType = valType; pVal->val = val; } +/** + * @brief Primary TS column not included. + * + * @param pRow + * @return FORCE_INLINE + */ static FORCE_INLINE col_id_t tdRowGetNCols(STSRow *pRow) { return *(col_id_t *)TD_ROW_NCOLS(pRow); } static FORCE_INLINE void tdRowCpy(void *dst, const STSRow *pRow) { memcpy(dst, pRow, TD_ROW_LEN(pRow)); } static FORCE_INLINE const char *tdRowEnd(STSRow *pRow) { return (const char *)POINTER_SHIFT(pRow, TD_ROW_LEN(pRow)); } @@ -250,7 +256,7 @@ static FORCE_INLINE void *tdGetBitmapAddrTp(STSRow *pRow, uint32_t flen) { static FORCE_INLINE void *tdGetBitmapAddrKv(STSRow *pRow, col_id_t nKvCols) { // The primary TS key is stored separatedly and is Norm value, thus should minus 1 firstly - return POINTER_SHIFT(TD_ROW_COL_IDX(pRow), (--nKvCols) * sizeof(SKvRowIdx)); + return POINTER_SHIFT(TD_ROW_COL_IDX(pRow), nKvCols * sizeof(SKvRowIdx)); } void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols); int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); @@ -322,10 +328,9 @@ typedef struct { void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema); int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow); -bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); -bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal); -bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); +bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal); +bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal); bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, SCellVal *pVal); bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6f58a0ded1..b13e5f1ca8 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -330,15 +330,17 @@ void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterReset(&iter, row); printf("%s >>>type:%d,sver:%d ", tag, (int32_t)TD_ROW_TYPE(row), (int32_t)TD_ROW_SVER(row)); - for (int i = 0; i < pSchema->numOfCols; ++i) { - STColumn *stCol = pSchema->columns + i; - SCellVal sVal = {255, NULL}; - if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { + + STColumn *cols = (STColumn *)&iter.pSchema->columns; + while (true) { + SCellVal sVal = {.valType = 255, NULL}; + if (!tdSTSRowIterNext(&iter, &sVal)) { break; } ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); - tdSCellValPrint(&sVal, stCol->type); + tdSCellValPrint(&sVal, cols[iter.colIdx - 1].type); } + printf("\n"); } @@ -425,7 +427,7 @@ bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVa tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); return true; } - int16_t nCols = tdRowGetNCols(pRow) - 1; + int16_t nCols = tdRowGetNCols(pRow); if (nCols <= 0) { pVal->valType = TD_VTYPE_NONE; return true; @@ -456,46 +458,41 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } -bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { + if (pIter->colIdx >= pIter->pSchema->numOfCols) { + return false; + } + + STColumn *pCol = &pIter->pSchema->columns[pIter->colIdx]; + + if (pCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { pVal->val = &pIter->pRow->ts; pVal->valType = TD_VTYPE_NORM; + ++pIter->colIdx; return true; } if (TD_IS_TP_ROW(pIter->pRow)) { - STColumn *pCol = NULL; - STSchema *pSchema = pIter->pSchema; - while (pIter->colIdx < pSchema->numOfCols) { - pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key - if (colId == pCol->colId) { - break; - } else if (pCol->colId < colId) { - ++pIter->colIdx; - continue; - } else { - return false; - } - } tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); - ++pIter->colIdx; } else if (TD_IS_KV_ROW(pIter->pRow)) { - return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal); + tdGetKvRowValOfColEx(pIter, pCol->colId, &pIter->kvIdx, pVal); } else { - pVal->valType = TD_VTYPE_NONE; - terrno = TSDB_CODE_INVALID_PARA; - if (COL_REACH_END(colId, pIter->maxColId)) return false; + ASSERT(0); } + ++pIter->colIdx; + return true; } -bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) { +bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal) { STSRow *pRow = pIter->pRow; SKvRowIdx *pKvIdx = NULL; bool colFound = false; - col_id_t kvNCols = tdRowGetNCols(pRow) - 1; + col_id_t kvNCols = tdRowGetNCols(pRow); + void *pColIdx = TD_ROW_COL_IDX(pRow); + while (*nIdx < kvNCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx)); + pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, *nIdx * sizeof(SKvRowIdx)); if (pKvIdx->colId == colId) { ++(*nIdx); pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); @@ -518,34 +515,37 @@ bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, } } -#ifdef TD_SUPPORT_BITMAP - int16_t colIdx = -1; - if (pKvIdx) colIdx = POINTER_DISTANCE(pKvIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); - if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + if (tdGetBitmapValType(pIter->pBitmap, (*nIdx) - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif return true; } bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { STSRow *pRow = pIter->pRow; - if (IS_VAR_DATA_TYPE(colType)) { - pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + if (pRow->statis == 0) { + pVal->valType = TD_VTYPE_NORM; + if (IS_VAR_DATA_TYPE(colType)) { + pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + return TSDB_CODE_SUCCESS; } -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; + return terrno; + } + + if (pVal->valType == TD_VTYPE_NORM) { + if (IS_VAR_DATA_TYPE(colType)) { + pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif return true; } @@ -625,7 +625,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { if (maxVarDataLen > 0) { varBuf = taosMemoryMalloc(maxVarDataLen); if (!varBuf) { - if(isAlloc) { + if (isAlloc) { taosMemoryFreeClear(*ppRow); } terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -673,45 +673,6 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { return 0; } -bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pVal->val = &pIter->pRow->ts; - pVal->valType = TD_VTYPE_NORM; - return true; - } - - STSRow *pRow = pIter->pRow; - int16_t colIdx = -1; - if (TD_IS_TP_ROW(pRow)) { - STSchema *pSchema = pIter->pSchema; - STColumn *pCol = - (STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ); - if (!pCol) { - pVal->valType = TD_VTYPE_NONE; - if (COL_REACH_END(colId, pIter->maxColId)) return false; - return true; - } -#ifdef TD_SUPPORT_BITMAP - colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); -#endif - tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); - } else if (TD_IS_KV_ROW(pRow)) { - SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), - compareKvRowColId, TD_EQ); -#ifdef TD_SUPPORT_BITMAP - if (pIdx) { - colIdx = POINTER_DISTANCE(pIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); - } -#endif - tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); - } else { - if (COL_REACH_END(colId, pIter->maxColId)) return false; - pVal->valType = TD_VTYPE_NONE; - } - - return true; -} - static int32_t tdCompareColId(const void *arg1, const void *arg2) { int32_t colId = *(int32_t *)arg1; STColumn *pCol = (STColumn *)arg2; @@ -853,7 +814,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP - TASSERT(colIdx < tdRowGetNCols(pRow) - 1); + TASSERT(colIdx < tdRowGetNCols(pRow)); if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { output->valType = TD_VTYPE_NONE; return terrno; @@ -938,7 +899,7 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp break; case TD_VTYPE_NONE: if (!pBuilder->hasNone) pBuilder->hasNone = true; - break; + return TSDB_CODE_SUCCESS; default: ASSERT(0); break; @@ -970,13 +931,11 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo STSRow *row = pBuilder->pBuf; // No need to store None/Null values. + SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); + pColIdx->colId = colId; + pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN if (valType == TD_VTYPE_NORM) { - // ts key stored in STSRow.ts - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); if (IS_VAR_DATA_TYPE(colType)) { if (isCopyVarData) { memcpy(ptr, val, varDataTLen(val)); @@ -987,26 +946,6 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo TD_ROW_LEN(row) += TYPE_BYTES[colType]; } } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - const void *nullVal = getNullValue(colType); - - if (IS_VAR_DATA_TYPE(colType)) { - if (isCopyVarData) { - memcpy(ptr, nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(ptr, nullVal, TYPE_BYTES[colType]); - TD_ROW_LEN(row) += TYPE_BYTES[colType]; - } - } -#endif return 0; } @@ -1044,24 +983,6 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), val, TYPE_BYTES[colType]); } } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - // TODO: Null value for new data types imported since 3.0 need to be defined. - const void *nullVal = getNullValue(colType); - if (IS_VAR_DATA_TYPE(colType)) { - // ts key stored in STSRow.ts - *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); - - if (isCopyVarData) { - memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), nullVal, TYPE_BYTES[colType]); - } - } -#endif return 0; } @@ -1133,14 +1054,14 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { break; case TD_ROW_KV: #ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols - 1); memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps); #endif len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + pBuilder->nBoundBitmaps; // add TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); - TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); + TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols - 1); break; default: TASSERT(0); @@ -1329,7 +1250,7 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { pIter->pRow = pRow; pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); pIter->offset = 0; - pIter->colIdx = PRIMARYKEY_TIMESTAMP_COL_ID; + pIter->colIdx = 0; pIter->kvIdx = 0; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 6ce8dbe5d9..ced98c8afa 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -325,7 +325,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { for (int32_t i = 0; i < colActual; i++) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SCellVal sVal = {0}; - if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { + if (!tdSTSRowIterNext(&iter, &sVal)) { break; } if (colDataAppend(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 468370b1a7..1da65288d0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -182,7 +182,7 @@ static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes); - tsdbInfo("-------------------%d\n", pCol->info.bytes); + // tsdbInfo("-------------------%d\n", pCol->info.bytes); } } diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 8f7ff04d92..e709643af9 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -13,38 +13,37 @@ * along with this program. If not, see . */ -#include "os.h" #include "tsimplehash.h" +#include "os.h" #include "taoserror.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 -#define HASH_MAX_CAPACITY (1024*1024*16) +#define HASH_MAX_CAPACITY (1024 * 1024 * 16) #define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) -#define GET_SHASH_NODE_KEY(_n, _dl) ((char*)(_n) + sizeof(SHNode) + (_dl)) -#define GET_SHASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHNode)) +#define GET_SHASH_NODE_KEY(_n, _dl) ((char *)(_n) + sizeof(SHNode) + (_dl)) +#define GET_SHASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHNode)) -#define HASH_INDEX(v, c) ((v) & ((c)-1)) -#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) +#define HASH_INDEX(v, c) ((v) & ((c)-1)) -#define FREE_HASH_NODE(_n) \ - do { \ - taosMemoryFreeClear(_n); \ +#define FREE_HASH_NODE(_n) \ + do { \ + taosMemoryFreeClear(_n); \ } while (0); typedef struct SHNode { - struct SHNode *next; - char data[]; + struct SHNode *next; + char data[]; } SHNode; struct SSHashObj { - SHNode **hashList; - size_t capacity; // number of slots - int64_t size; // number of elements in hash table - _hash_fn_t hashFp; // hash function - _equal_fn_t equalFp; // equal function - int32_t keyLen; - int32_t dataLen; + SHNode **hashList; + size_t capacity; // number of slots + int64_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function + int32_t keyLen; + int32_t dataLen; }; static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { @@ -62,7 +61,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t capacity = 4; } - SSHashObj* pHashObj = (SSHashObj*) taosMemoryCalloc(1, sizeof(SSHashObj)); + SSHashObj *pHashObj = (SSHashObj *)taosMemoryCalloc(1, sizeof(SSHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -72,7 +71,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t pHashObj->capacity = taosHashCapacity((int32_t)capacity); pHashObj->equalFp = memcmp; - pHashObj->hashFp = fn; + pHashObj->hashFp = fn; ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); pHashObj->keyLen = keyLen; @@ -91,7 +90,7 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { if (pHashObj == NULL) { return 0; } - return (int32_t)atomic_load_64((int64_t*)&pHashObj->size); + return (int32_t)atomic_load_64((int64_t *)&pHashObj->size); } static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { @@ -108,41 +107,42 @@ static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pDat } static void taosHashTableResize(SSHashObj *pHashObj) { - if (!HASH_NEED_RESIZE(pHashObj)) { + if (!SHASH_NEED_RESIZE(pHashObj)) { return; } int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); if (newCapacity > HASH_MAX_CAPACITY) { -// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", -// pHashObj->capacity, HASH_MAX_CAPACITY); + // uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", + // pHashObj->capacity, HASH_MAX_CAPACITY); return; } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { -// qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); + // qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; } size_t inc = newCapacity - pHashObj->capacity; - memset((char*)pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc); + memset((char *)pNewEntryList + pHashObj->capacity * sizeof(void *), 0, inc); pHashObj->hashList = pNewEntryList; pHashObj->capacity = newCapacity; for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { - SHNode* pNode = pHashObj->hashList[idx]; - SHNode *pNext; - SHNode *pPrev = NULL; - + SHNode *pNode = pHashObj->hashList[idx]; if (pNode == NULL) { continue; } + SHNode *pNext; + SHNode *pPrev = NULL; + + while (pNode != NULL) { - void* key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen); + void *key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen); uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->dataLen); int32_t newIdx = HASH_INDEX(hashVal, pHashObj->capacity); @@ -166,8 +166,9 @@ static void taosHashTableResize(SSHashObj *pHashObj) { int64_t et = taosGetTimestampUs(); -// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, -// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); + // uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", + // (int32_t)pHashObj->capacity, + // ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { @@ -210,7 +211,7 @@ int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { pNewNode->next = pHashObj->hashList[slot]; pHashObj->hashList[slot] = pNewNode; atomic_add_fetch_64(&pHashObj->size, 1); - } else { //update data + } else { // update data memcpy(GET_SHASH_NODE_DATA(pNode), data, pHashObj->dataLen); } @@ -230,9 +231,7 @@ static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void return pNode; } -static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) { - return tSimpleHashGetSize(pHashObj) == 0; -} +static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) { return tSimpleHashGetSize(pHashObj) == 0; } void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL) { @@ -299,9 +298,9 @@ size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { return (pHashObj->capacity * sizeof(void *)) + sizeof(SHNode) * tSimpleHashGetSize(pHashObj) + sizeof(SSHashObj); } -void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen) { +void *tSimpleHashGetKey(const SSHashObj *pHashObj, void *data, size_t *keyLen) { int32_t offset = offsetof(SHNode, data); - SHNode *node = ((SHNode*)(char*)data - offset); + SHNode *node = ((SHNode *)(char *)data - offset); if (keyLen != NULL) { *keyLen = pHashObj->keyLen; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 9c9807f843..b7532173c8 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1192,7 +1192,10 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, pBuilder->hasNone = true; } + tdSRowEnd(pBuilder); + *gotRow = true; + #ifdef TD_DEBUG_PRINT_ROW STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1); tdSRowPrint(row, pSTSchema, __func__); @@ -1201,7 +1204,6 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, } // *len = pBuilder->extendedRowSize; - tdSRowEnd(pBuilder); return TSDB_CODE_SUCCESS; } From 4cd1452e61843ea9fd1e3735538d67d1c8efd08b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 10 Aug 2022 17:55:04 +0800 Subject: [PATCH 128/165] fix(stream): check udf --- source/libs/executor/src/timewindowoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9198862ba3..610303dc50 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1695,7 +1695,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { for (int32_t i = 0; i < numOfCols; i++) { - if (!fmIsInvertible(pFCtx[i].functionId)) { + if (fmIsUserDefinedFunc(pFCtx[i].functionId) || !fmIsInvertible(pFCtx[i].functionId)) { return false; } } From 01bac29880ec96e7fd1a5a277388658fa1d20335 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 10 Aug 2022 17:57:45 +0800 Subject: [PATCH 129/165] update case about diff --- tests/system-test/2-query/function_diff.py | 125 +++++++++------------ 1 file changed, 51 insertions(+), 74 deletions(-) diff --git a/tests/system-test/2-query/function_diff.py b/tests/system-test/2-query/function_diff.py index 2f463e59a0..8cc6a8544f 100644 --- a/tests/system-test/2-query/function_diff.py +++ b/tests/system-test/2-query/function_diff.py @@ -30,7 +30,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) - def diff_query_form(self, col="c1", alias="", table_expr="t1", condition=""): + def diff_query_form(self, col="c1", alias="", table_expr="db.t1", condition=""): ''' diff function: @@ -44,7 +44,7 @@ class TDTestCase: return f"select diff({col}) {alias} from {table_expr} {condition}" - def checkdiff(self,col="c1", alias="", table_expr="t1", condition="" ): + def checkdiff(self,col="c1", alias="", table_expr="db.t1", condition="" ): line = sys._getframe().f_back.f_lineno pre_sql = self.diff_query_form( col=col, table_expr=table_expr, condition=condition @@ -60,7 +60,7 @@ class TDTestCase: return if "order by tbname" in condition: - tdSql.error(self.diff_query_form( + tdSql.query(self.diff_query_form( col=col, alias=alias, table_expr=table_expr, condition=condition )) return @@ -164,9 +164,9 @@ class TDTestCase: self.checkdiff(**case6) # case7~8: nested query - # case7 = {"table_expr": "(select c1 from stb1)"} + # case7 = {"table_expr": "(select c1 from db.stb1)"} # self.checkdiff(**case7) - # case8 = {"table_expr": "(select diff(c1) c1 from stb1 group by tbname)"} + # case8 = {"table_expr": "(select diff(c1) c1 from db.stb1 group by tbname)"} # self.checkdiff(**case8) # case9~10: mix with tbname/ts/tag/col @@ -200,15 +200,15 @@ class TDTestCase: self.checkdiff(**case17) # case18~19: with group by # case18 = { - # "table_expr": "t1", + # "table_expr": "db.t1", # "condition": "group by c6" # } # self.checkdiff(**case18) - # case19 = { - # "table_expr": "stb1", - # "condition": "partition by tbname" # partition by tbname - # } - # self.checkdiff(**case19) + case19 = { + "table_expr": "db.stb1", + "condition": "partition by tbname order by tbname" # partition by tbname + } + self.checkdiff(**case19) # # case20~21: with order by # case20 = {"condition": "order by ts"} @@ -226,7 +226,7 @@ class TDTestCase: } self.checkdiff(**case23) # case24 = { - # "table_expr": "stb1", + # "table_expr": "db.stb1", # "condition": "group by tbname slimit 1 soffset 1" # } # self.checkdiff(**case24) @@ -241,13 +241,13 @@ class TDTestCase: # # form test tdSql.error(self.diff_query_form(col="")) # no col - tdSql.error("diff(c1) from stb1") # no select - tdSql.error("select diff from t1") # no diff condition - tdSql.error("select diff c1 from t1") # no brackets - tdSql.error("select diff(c1) t1") # no from + tdSql.error("diff(c1) from db.stb1") # no select + tdSql.error("select diff from db.t1") # no diff condition + tdSql.error("select diff c1 from db.t1") # no brackets + tdSql.error("select diff(c1) db.t1") # no from tdSql.error("select diff( c1 ) from ") # no table_expr # tdSql.error(self.diff_query_form(col="st1")) # tag col - tdSql.query("select diff(st1) from t1 ") + tdSql.query("select diff(st1) from db.t1") # tdSql.error(self.diff_query_form(col=1)) # col is a value tdSql.error(self.diff_query_form(col="'c1'")) # col is a string tdSql.error(self.diff_query_form(col=None)) # col is NULL 1 @@ -260,18 +260,18 @@ class TDTestCase: tdSql.error(self.diff_query_form(col='ts')) # Primary key tdSql.error(self.diff_query_form(col='avg(c1)')) # expr col # tdSql.error(self.diff_query_form(col='c6')) # bool col - tdSql.query("select diff(c6) from t1") + tdSql.query("select diff(c6) from db.t1") tdSql.error(self.diff_query_form(col='c4')) # binary col tdSql.error(self.diff_query_form(col='c10')) # nachr col tdSql.error(self.diff_query_form(col='c10')) # not table_expr col - tdSql.error(self.diff_query_form(col='t1')) # tbname - tdSql.error(self.diff_query_form(col='stb1')) # stbname + tdSql.error(self.diff_query_form(col='db.t1')) # tbname + tdSql.error(self.diff_query_form(col='db.stb1')) # stbname tdSql.error(self.diff_query_form(col='db')) # datbasename # tdSql.error(self.diff_query_form(col=True)) # col is BOOL 1 # tdSql.error(self.diff_query_form(col='True')) # col is BOOL 2 tdSql.error(self.diff_query_form(col='*')) # col is all col - tdSql.error("select diff[c1] from t1") # sql form error 1 - tdSql.error("select diff{c1} from t1") # sql form error 2 + tdSql.error("select diff[c1] from db.t1") # sql form error 1 + tdSql.error("select diff{c1} from db.t1") # sql form error 2 tdSql.error(self.diff_query_form(col="[c1]")) # sql form error 3 # tdSql.error(self.diff_query_form(col="c1, c2")) # sql form error 3 # tdSql.error(self.diff_query_form(col="c1, 2")) # sql form error 3 @@ -284,7 +284,7 @@ class TDTestCase: # tdSql.error(self.diff_query_form(alias=" + 2")) # mix with arithmetic 1 tdSql.error(self.diff_query_form(alias=" + avg(c1)")) # mix with arithmetic 2 tdSql.query(self.diff_query_form(alias=", c2")) # mix with other 1 - # tdSql.error(self.diff_query_form(table_expr="stb1")) # select stb directly + # tdSql.error(self.diff_query_form(table_expr="db.stb1")) # select stb directly stb_join = { "col": "stb1.c1", "table_expr": "stb1, stb2", @@ -296,17 +296,17 @@ class TDTestCase: } tdSql.error(self.diff_query_form(**interval_sql)) # interval group_normal_col = { - "table_expr": "t1", + "table_expr": "db.t1", "condition": "group by c6" } tdSql.error(self.diff_query_form(**group_normal_col)) # group by normal col slimit_soffset_sql = { - "table_expr": "stb1", + "table_expr": "db.stb1", "condition": "group by tbname slimit 1 soffset 1" } # tdSql.error(self.diff_query_form(**slimit_soffset_sql)) order_by_tbname_sql = { - "table_expr": "stb1", + "table_expr": "db.stb1", "condition": "group by tbname order by tbname" } tdSql.error(self.diff_query_form(**order_by_tbname_sql)) @@ -351,63 +351,40 @@ class TDTestCase: "create stable db.stb2 (ts timestamp, c1 int) tags(st2 int)" ) for i in range(tbnum): - tdSql.execute(f"create table t{i} using stb1 tags({i})") - tdSql.execute(f"create table tt{i} using stb2 tags({i})") + tdSql.execute(f"create table t{i} using db.stb1 tags({i})") + tdSql.execute(f"create table tt{i} using db.stb2 tags({i})") pass def diff_support_stable(self): - tdSql.query(" select diff(1) from stb1 ") + tdSql.query(" select diff(1) from db.stb1 ") tdSql.checkRows(229) tdSql.checkData(0,0,0) - tdSql.query("select diff(c1) from stb1 partition by tbname ") + tdSql.query("select diff(c1) from db.stb1 partition by tbname ") tdSql.checkRows(190) - # tdSql.query("select diff(st1) from stb1 partition by tbname") - # tdSql.checkRows(229) - tdSql.query("select diff(st1+c1) from stb1 partition by tbname") + + tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname") tdSql.checkRows(190) - tdSql.query("select diff(st1+c1) from stb1 partition by tbname") + tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname") tdSql.checkRows(190) - tdSql.query("select diff(st1+c1) from stb1 partition by tbname") - tdSql.checkRows(190) - - # # bug need fix - # tdSql.query("select diff(st1+c1) from stb1 partition by tbname slimit 1 ") - # tdSql.checkRows(19) - # tdSql.error("select diff(st1+c1) from stb1 partition by tbname limit 1 ") - - - # bug need fix - tdSql.query("select diff(st1+c1) from stb1 partition by tbname") + tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname") tdSql.checkRows(190) # bug need fix - # tdSql.query("select tbname , diff(c1) from stb1 partition by tbname") - # tdSql.checkRows(199) - # tdSql.query("select tbname , diff(st1) from stb1 partition by tbname") - # tdSql.checkRows(199) - # tdSql.query("select tbname , diff(st1) from stb1 partition by tbname slimit 1") - # tdSql.checkRows(19) + tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname") + tdSql.checkRows(190) + + # bug need fix + tdSql.query("select tbname , diff(c1) from db.stb1 partition by tbname") + tdSql.checkRows(190) + tdSql.query("select tbname , diff(st1) from db.stb1 partition by tbname") + tdSql.checkRows(220) + # partition by tags - # tdSql.query("select st1 , diff(c1) from stb1 partition by st1") - # tdSql.checkRows(199) - # tdSql.query("select diff(c1) from stb1 partition by st1") - # tdSql.checkRows(199) - # tdSql.query("select st1 , diff(c1) from stb1 partition by st1 slimit 1") - # tdSql.checkRows(19) - # tdSql.query("select diff(c1) from stb1 partition by st1 slimit 1") - # tdSql.checkRows(19) - - # partition by col - # tdSql.query("select c1 , diff(c1) from stb1 partition by c1") - # tdSql.checkRows(199) - # tdSql.query("select diff(c1) from stb1 partition by c1") - # tdSql.checkRows(41) - # tdSql.query("select c1 , diff(c1) from stb1 partition by st1 slimit 1") - # tdSql.checkRows(19) - # tdSql.query("select diff(c1) from stb1 partition by st1 slimit 1") - # tdSql.checkRows(19) - + tdSql.query("select st1 , diff(c1) from db.stb1 partition by st1") + tdSql.checkRows(190) + tdSql.query("select diff(c1) from db.stb1 partition by st1") + tdSql.checkRows(190) def diff_test_run(self) : @@ -430,18 +407,18 @@ class TDTestCase: tdLog.printNoPrefix("######## insert data in the range near the max(bigint/double):") self.diff_test_table(tbnum) - tdSql.execute(f"insert into t1(ts, c1,c2,c5,c7) values " + tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values " f"({nowtime - (per_table_rows + 1) * 10}, {2**31-1}, {3.4*10**38}, {1.7*10**308}, {2**63-1})") - tdSql.execute(f"insert into t1(ts, c1,c2,c5,c7) values " + tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values " f"({nowtime - (per_table_rows + 2) * 10}, {2**31-1}, {3.4*10**38}, {1.7*10**308}, {2**63-1})") self.diff_current_query() self.diff_error_query() tdLog.printNoPrefix("######## insert data in the range near the min(bigint/double):") self.diff_test_table(tbnum) - tdSql.execute(f"insert into t1(ts, c1,c2,c5,c7) values " + tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values " f"({nowtime - (per_table_rows + 1) * 10}, {1-2**31}, {-3.4*10**38}, {-1.7*10**308}, {1-2**63})") - tdSql.execute(f"insert into t1(ts, c1,c2,c5,c7) values " + tdSql.execute(f"insert into db.t1(ts, c1,c2,c5,c7) values " f"({nowtime - (per_table_rows + 2) * 10}, {1-2**31}, {-3.4*10**38}, {-1.7*10**308}, {512-2**63})") self.diff_current_query() self.diff_error_query() From 0801dc4f1d4de4659576fac1de2fd0770276e1df Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 10 Aug 2022 17:59:02 +0800 Subject: [PATCH 130/165] doc: add performance_schema page --- docs/zh/12-taos-sql/14-stream.md | 12 --- docs/zh/12-taos-sql/23-perf.md | 129 +++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 12 deletions(-) create mode 100644 docs/zh/12-taos-sql/23-perf.md diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 7ff7da2bfb..024742ed8b 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -89,10 +89,6 @@ T = 最新事件时间 - watermark 无论在哪种模式下，watermark 都应该被妥善设置，来得到正确结果（直接丢弃模式）或避免频繁触发重算带来的性能开销（重新计算模式）。 -## 流式计算的数据填充策略 - -TODO - ## 流式计算与会话窗口（session window） ```sql @@ -105,14 +101,6 @@ window_clause: { 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 -## 流式计算的监控与流任务分布查询 - -TODO - -## 流式计算的内存控制与存算分离 - -TODO - ## 流式计算的暂停与恢复 ```sql diff --git a/docs/zh/12-taos-sql/23-perf.md b/docs/zh/12-taos-sql/23-perf.md new file mode 100644 index 0000000000..4c7660e829 --- /dev/null +++ b/docs/zh/12-taos-sql/23-perf.md @@ -0,0 +1,129 @@ +--- +sidebar_label: 性能数据库 +title: 性能数据库 +--- + +TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其中存储了与性能有关的统计数据。本节详细介绍其中的表和详细的表结构。 + +## PERF_APP + +提供接入集群的应用（客户端）的相关信息。也可以使用 SHOW APPS 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | ------------------------------- | +| 1 | app_id | UBIGINT | 客户端 ID | +| 2 | ip | BINARY(16) | 客户端地址 | +| 3 | pid | INT | 客户端进程 号 | +| 4 | name | BINARY(24) | 客户端名称 | +| 5 | start_time | TIMESTAMP | 客户端启动时间 | +| 6 | insert_req | UBIGINT | insert 请求次数 | +| 7 | insert_row | UBIGINT | insert 插入行数 | +| 8 | insert_time | UBIGINT | insert 请求的处理时间，单位微秒 | +| 9 | insert_bytes | UBIGINT | insert 请求消息字节数 | +| 10 | fetch_bytes | UBIGINT | 查询结果字节数 | +| 11 | query_time | UBIGINT | 查询请求处理时间 | +| 12 | slow_query | UBIGINT | 慢查询（处理时间 >= 3 秒）个数 | +| 13 | total_req | UBIGINT | 总请求数 | +| 14 | current_req | UBIGINT | 当前正在处理的请求个数 | +| 15 | last_access | TIMESTAMP | 最后更新时间 | + +## PERF_CONNECTIONS + +数据库的连接的相关信息。也可以使用 SHOW CONNECTIONS 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | -------------------------------------------------- | +| 1 | conn_id | INT | 连接 ID | +| 2 | user | BINARY(24) | 用户名 | +| 3 | app | BINARY(24) | 客户端名称 | +| 4 | pid | UINT | 发起此连接的客户端在自己所在服务器或主机上的进程号 | +| 5 | end_point | BINARY(128) | 客户端地址 | +| 6 | login_time | TIMESTAMP | 登录时间 | +| 7 | last_access | TIMESTAMP | 最后更新时间 | + +## PERF_QUERIES + +提供当前正在执行的 SQL 语句的信息。也可以使用 SHOW QUERIES 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | ---------------------------- | +| 1 | kill_id | UBIGINT | 用来停止查询的 ID | +| 2 | query_id | INT | 查询 ID | +| 3 | conn_id | UINT | 连接 ID | +| 4 | app | BINARY(24) | app 名称 | +| 5 | pid | INT | app 在自己所在主机上的进程号 | +| 6 | user | BINARY(24) | 用户名 | +| 7 | end_point | BINARY(16) | 客户端地址 | +| 8 | create_time | TIMESTAMP | 创建时间 | +| 9 | exec_usec | BIGINT | 已执行时间 | +| 10 | stable_query | BOOL | 是否是超级表查询 | +| 11 | sub_num | INT | 子查询数量 | +| 12 | sub_status | BINARY(1000) | 子查询状态 | +| 13 | sql | BINARY(1024) | SQL 语句 | + +## PERF_TOPICS + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ------------------------------ | +| 1 | topic_name | BINARY(192) | topic 名称 | +| 2 | db_name | BINARY(64) | topic 相关的 DB | +| 3 | create_time | TIMESTAMP | topic 的 创建时间 | +| 4 | sql | BINARY(1024) | 创建该 topic 时所用的 SQL 语句 | + +## PERF_CONSUMERS + +| # | **列名** | **数据类型** | **说明** | +| --- | :------------: | ------------ | ----------------------------------------------------------- | +| 1 | consumer_id | BIGINT | 消费者的唯一 ID | +| 2 | consumer_group | BINARY(192) | 消费者组 | +| 3 | client_id | BINARY(192) | 用户自定义字符串，通过创建 consumer 时指定 client_id 来展示 | +| 4 | status | BINARY(20) | 消费者当前状态 | +| 5 | topics | BINARY(204) | 被订阅的 topic。若订阅多个 topic，则展示为多行 | +| 6 | up_time | TIMESTAMP | 第一次连接 taosd 的时间 | +| 7 | subscribe_time | TIMESTAMP | 上一次发起订阅的时间 | +| 8 | rebalance_time | TIMESTAMP | 上一次触发 rebalance 的时间 | + +## PERF_SUBSCRIPTIONS + +| # | **列名** | **数据类型** | **说明** | +| --- | :------------: | ------------ | ------------------------ | +| 1 | topic_name | BINARY(204) | 被订阅的 topic | +| 2 | consumer_group | BINARY(193) | 订阅者的消费者组 | +| 3 | vgroup_id | INT | 消费者被分配的 vgroup id | +| 4 | consumer_id | BIGINT | 消费者的唯一 id | + +## PERF_TRANS + +| # | **列名** | **数据类型** | **说明** | +| --- | :--------------: | ------------ | -------- | +| 1 | id | INT | | +| 2 | create_time | TIMESTAMP | | +| 3 | stage | BINARY(12) | | +| 4 | db1 | BINARY(64) | | +| 5 | db2 | BINARY(64) | | +| 6 | failed_times | INT | | +| 7 | last_exec_time | TIMESTAMP | | +| 8 | last_action_info | BINARY(511) | | + +## PERF_SMAS + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ------------------------------------------- | +| 1 | sma_name | BINARY(192) | 时间维度的预计算 (time-range-wise sma) 名称 | +| 2 | create_time | TIMESTAMP | sma 创建时间 | +| 3 | stable_name | BINARY(192) | sma 所属的超级表名称 | +| 4 | vgroup_id | INT | sma 专属的 vgroup 名称 | + +## PERF_STREAMS + +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | --------------------------------------- | +| 1 | stream_name | BINARY(64) | 流计算名称 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | sql | BINARY(1024) | 创建流计算时提供的 SQL 语句 | +| 4 | status | BIANRY(20) | 流当前状态 | +| 5 | source_db | BINARY(64) | 源数据库 | +| 6 | target_db | BIANRY(64) | 目的数据库 | +| 7 | target_table | BINARY(192) | 流计算写入的目标表 | +| 8 | watermark | BIGINT | watermark，详见 SQL 手册流式计算 | +| 9 | trigger | INT | 计算结果推送模式，详见 SQL 手册流式计算 | From 668a78a79f367f83bf8ee2c3e238925ff5350194 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 10 Aug 2022 18:02:43 +0800 Subject: [PATCH 131/165] fix: fix histogram not setting content when process all null columns --- source/libs/function/src/builtinsimpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 9aa7c11fce..b325848f11 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -4281,9 +4281,9 @@ static int32_t histogramFunctionImpl(SqlFunctionCtx* pCtx, bool isPartial) { } if (!isPartial) { - SET_VAL(GET_RES_INFO(pCtx), numOfElems, pInfo->numOfBins); + GET_RES_INFO(pCtx)->numOfRes = pInfo->numOfBins; } else { - SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + GET_RES_INFO(pCtx)->numOfRes = 1; } return TSDB_CODE_SUCCESS; } From 9e5d3473832152ee4637ae3952de880f27641b5a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 10 Aug 2022 18:22:22 +0800 Subject: [PATCH 132/165] enh(tmq): enable background heartbeat by default --- source/client/src/tmq.c | 1 + source/dnode/mnode/impl/src/mndStream.c | 18 +++++++++++++++--- source/util/src/terror.c | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index c0874caf95..28c359b2fb 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -212,6 +212,7 @@ tmq_conf_t* tmq_conf_new() { conf->autoCommit = true; conf->autoCommitInterval = 5000; conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; + conf->hbBgEnable = true; return conf; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0f20188033..b172b00fab 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -929,14 +929,26 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->status, true); + char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pStream->sourceDb, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&n, varDataVal(sourceDB)); + varDataSetLen(sourceDB, strlen(varDataVal(sourceDB))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->sourceDb, true); + colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false); + char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pStream->targetDb, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&n, varDataVal(targetDB)); + varDataSetLen(targetDB, strlen(varDataVal(targetDB))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetDb, true); + colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false); + char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n)); + varDataSetLen(targetSTB, strlen(varDataVal(targetSTB))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetSTbName, true); + colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index bb26f9b2f7..8a3d0620ed 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -281,7 +281,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC, "Invalid topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_QUERY, "Topic with invalid query") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION, "Topic with invalid option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_EXIST, "Consumer not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED, "Consumer unchanged") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED, "Topic unchanged") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST, "Subcribe not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_OFFSET_NOT_EXIST, "Offset not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer not ready") From 365dc43fec708e6f26e2cfbc2e293510f69ec073 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 10 Aug 2022 18:53:29 +0800 Subject: [PATCH 133/165] enh(tmq): add log --- source/dnode/mnode/impl/src/mndSubscribe.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 8feed476cd..3f310ee9c0 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -398,13 +398,27 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } } - // 8. TODO generate logs - mInfo("rebalance calculation completed, rebalanced vg:"); + // 8. generate logs + mInfo("mq rebalance: calculation completed, rebalanced vg:"); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); - mInfo("vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId, + mInfo("mq rebalance: vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } + { + void *pIter = NULL; + while (1) { + pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); + if (pIter == NULL) break; + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; + int32_t sz = taosArrayGetSize(pConsumerEp->vgs); + mInfo("mq rebalance: final cfg: consumer %ld has %d vg", pConsumerEp->consumerId, sz); + for (int32_t i = 0; i < sz; i++) { + SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); + mInfo("mq rebalance: final cfg: vg %d to consumer %ld", pVgEp->vgId, pConsumerEp->consumerId); + } + } + } // 9. clear taosHashCleanup(pHash); From bed9ca4d0854c21947b6ee862bbd7055bfeccf32 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 19:37:10 +0800 Subject: [PATCH 134/165] doc: change 3.0 doc --- docs/en/14-reference/08-taos-shell.md | 35 +++++++++-------- .../14-reference/09-support-platform/index.md | 11 +++--- docs/en/14-reference/12-directory.md | 1 - docs/zh/14-reference/08-taos-shell.md | 39 ++++++++++--------- .../14-reference/09-support-platform/index.md | 17 +++----- docs/zh/14-reference/12-directory.md | 1 - 6 files changed, 48 insertions(+), 56 deletions(-) diff --git a/docs/en/14-reference/08-taos-shell.md b/docs/en/14-reference/08-taos-shell.md index 002b515093..c947e86d1c 100644 --- a/docs/en/14-reference/08-taos-shell.md +++ b/docs/en/14-reference/08-taos-shell.md @@ -47,27 +47,28 @@ If the displayed content is followed by `...` you can use this command to change You can change the behavior of TDengine CLI by specifying command-line parameters. The following parameters are commonly used. -- -h, --host=HOST: FQDN of the server where the TDengine server is to be connected. Default is to connect to the local service -- -P, --port=PORT: Specify the port number to be used by the server. Default is `6030` -- -u, --user=USER: the user name to use when connecting. Default is `root` -- -p, --password=PASSWORD: the password to use when connecting to the server. Default is `taosdata` +- -h HOST: FQDN of the server where the TDengine server is to be connected. Default is to connect to the local service +- -P PORT: Specify the port number to be used by the server. Default is `6030` +- -u USER: the user name to use when connecting. Default is `root` +- -p PASSWORD: the password to use when connecting to the server. Default is `taosdata` - -?, --help: print out all command-line arguments And many more parameters. -- -c, --config-dir: Specify the directory where configuration file exists. The default is `/etc/taos`, and the default name of the configuration file in this directory is `taos.cfg` -- -C, --dump-config: Print the configuration parameters of `taos.cfg` in the default directory or specified by -c -- -d, --database=DATABASE: Specify the database to use when connecting to the server -- -D, --directory=DIRECTORY: Import the SQL script file in the specified path -- -f, --file=FILE: Execute the SQL script file in non-interactive mode -- -k, --check=CHECK: Specify the table to be checked -- -l, --pktlen=PKTLEN: Test package size to be used for network testing -- -n, --netrole=NETROLE: test scope for network connection test, default is `startup`. The value can be `client`, `server`, `rpc`, `startup`, `sync`, `speed`, or `fqdn`. -- -r, --raw-time: output the timestamp format as unsigned 64-bits integer (uint64_t in C language) -- -s, --commands=COMMAND: execute SQL commands in non-interactive mode -- -S, --pkttype=PKTTYPE: Specify the packet type used for network testing. The default is TCP, can be specified as either TCP or UDP when `speed` is specified to `netrole` parameter -- -T, --thread=THREADNUM: The number of threads to import data in multi-threaded mode -- -s, --commands: Run TDengine CLI commands without entering the terminal +- -a AUTHSTR: The auth string to use when connecting to the server +- -A: Generate auth string from password +- -c CONFIGDIR: Specify the directory where configuration file exists. The default is `/etc/taos`, and the default name of the configuration file in this directory is `taos.cfg` +- -C: Print the configuration parameters of `taos.cfg` in the default directory or specified by -c +- -d DATABASE: Specify the database to use when connecting to the server +- -f FILE: Execute the SQL script file in non-interactive mode +- -k: Check the service status, 0: unavailable，1: network ok，2: service ok，3: service degraded，4: exiting +- -l PKTLEN: Test package length to be used for network testing +- -n NETROLE: test scope for network connection test, default is `client`. The value can be `client`, `server` +- -N PKTNUM: Test package numbers to be used for network testing +- -r: output the timestamp format as unsigned 64-bits integer (uint64_t in C language) +- -s COMMAND: execute SQL commands in non-interactive mode +- -t: Check the details of the service status，status same as -k +- -w DISPLAYWIDTH: 客户端列显示宽度 - -z, --timezone=TIMEZONE: Specify time zone. Default is the value of current configuration file - -V, --version: Print out the current version number diff --git a/docs/en/14-reference/09-support-platform/index.md b/docs/en/14-reference/09-support-platform/index.md index af15758225..656344eff4 100644 --- a/docs/en/14-reference/09-support-platform/index.md +++ b/docs/en/14-reference/09-support-platform/index.md @@ -5,12 +5,11 @@ description: "List of platforms supported by TDengine server, client, and connec ## List of supported platforms for TDengine server -| | **CentOS 7/8** | **Ubuntu 16/18/20** | **Other Linux** | -| ------------ | -------------- | ------------------- | --------------- | -| X64 | ● | ● | | -| MIPS64 | | | ● | -| ARM64 | | ○ | ○ | -| Alpha64 | | | ○ | +| | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **Other Linux** | **UOS** | **Kylin** | **Ningsi V60/V80** | **HUAWEI EulerOS** | +| ------------------ | ----------------- | ---------------- | ---------------- | --------------- | ------- | --------- | ------------------ | ------------------ | +| X64 | ● | ● | ● | | ● | ● | ● | | +| Raspberry Pi ARM64 | | | | ● | | | | | +| HUAWEI cloud ARM64 | | | | | | | | ● | Note: ● means officially tested and verified, ○ means unofficially tested and verified. diff --git a/docs/en/14-reference/12-directory.md b/docs/en/14-reference/12-directory.md index d6cffd22e0..118bce8037 100644 --- a/docs/en/14-reference/12-directory.md +++ b/docs/en/14-reference/12-directory.md @@ -25,7 +25,6 @@ All executable files of TDengine are in the _/usr/local/taos/bin_ directory by d - _taosBenchmark_: TDengine testing tool - _remove.sh_: script to uninstall TDengine, please execute it carefully, link to the **rmtaos** command in the /usr/bin directory. Will remove the TDengine installation directory `/usr/local/taos`, but will keep `/etc/taos`, `/var/lib/taos`, `/var/log/taos` - _taosadapter_: server-side executable that provides RESTful services and accepts writing requests from a variety of other softwares -- _tarbitrator_: provides arbitration for two-node cluster deployments - _TDinsight.sh_: script to download TDinsight and install it - _set_core.sh_: script for setting up the system to generate core dump files for easy debugging - _taosd-dump-cfg.gdb_: script to facilitate debugging of taosd's gdb execution. diff --git a/docs/zh/14-reference/08-taos-shell.md b/docs/zh/14-reference/08-taos-shell.md index d4a516d2d2..2f3b551502 100644 --- a/docs/zh/14-reference/08-taos-shell.md +++ b/docs/zh/14-reference/08-taos-shell.md @@ -48,29 +48,30 @@ taos> SET MAX_BINARY_DISPLAY_WIDTH ; 您可通过配置命令行参数来改变 TDengine CLI 的行为。以下为常用的几个命令行参数： -- -h, --host=HOST: 要连接的 TDengine 服务端所在服务器的 FQDN, 默认为连接本地服务 -- -P, --port=PORT: 指定服务端所用端口号 -- -u, --user=USER: 连接时使用的用户名 -- -p, --password=PASSWORD: 连接服务端时使用的密码 +- -h HOST: 要连接的 TDengine 服务端所在服务器的 FQDN, 默认为连接本地服务 +- -P PORT: 指定服务端所用端口号 +- -u USER: 连接时使用的用户名 +- -p PASSWORD: 连接服务端时使用的密码 - -?, --help: 打印出所有命令行参数 还有更多其他参数： -- -c, --config-dir: 指定配置文件目录，Linux 环境下默认为 `/etc/taos`，该目录下的配置文件默认名称为 `taos.cfg` -- -C, --dump-config: 打印 -c 指定的目录中 `taos.cfg` 的配置参数 -- -d, --database=DATABASE: 指定连接到服务端时使用的数据库 -- -D, --directory=DIRECTORY: 导入指定路径中的 SQL 脚本文件 -- -f, --file=FILE: 以非交互模式执行 SQL 脚本文件。文件中一个 SQL 语句只能占一行 -- -k, --check=CHECK: 指定要检查的表 -- -l, --pktlen=PKTLEN: 网络测试时使用的测试包大小 -- -n, --netrole=NETROLE: 网络连接测试时的测试范围，默认为 `startup`, 可选值为 `client`、`server`、`rpc`、`startup`、`sync`、`speed` 和 `fqdn` 之一 -- -r, --raw-time: 将时间输出出无符号 64 位整数类型(即 C 语音中 uint64_t) -- -s, --commands=COMMAND: 以非交互模式执行的 SQL 命令 -- -S, --pkttype=PKTTYPE: 指定网络测试所用的包类型，默认为 TCP。只有 netrole 为 `speed` 时既可以指定为 TCP 也可以指定为 UDP -- -T, --thread=THREADNUM: 以多线程模式导入数据时的线程数 -- -s, --commands: 在不进入终端的情况下运行 TDengine 命令 -- -z, --timezone=TIMEZONE: 指定时区，默认为本地时区 -- -V, --version: 打印出当前版本号 +- -a AUTHSTR: 连接服务端的授权信息 +- -A: 通过用户名和密码计算授权信息 +- -c CONFIGDIR: 指定配置文件目录，Linux 环境下默认为 `/etc/taos`，该目录下的配置文件默认名称为 `taos.cfg` +- -C: 打印 -c 指定的目录中 `taos.cfg` 的配置参数 +- -d DATABASE: 指定连接到服务端时使用的数据库 +- -f FILE: 以非交互模式执行 SQL 脚本文件。文件中一个 SQL 语句只能占一行 +- -k: 测试服务端运行状态，0: unavailable，1: network ok，2: service ok，3: service degraded，4: exiting +- -l PKTLEN: 网络测试时使用的测试包大小 +- -n NETROLE: 网络连接测试时的测试范围，默认为 `client`, 可选值为 `client`、`server` +- -N PKTNUM: 网络测试时使用的测试包数量 +- -r: 将时间输出出无符号 64 位整数类型(即 C 语音中 uint64_t) +- -s COMMAND: 以非交互模式执行的 SQL 命令 +- -t: 测试服务端启动状态，状态同-k +- -w DISPLAYWIDTH: 客户端列显示宽度 +- -z TIMEZONE: 指定时区，默认为本地时区 +- -V: 打印出当前版本号 示例： diff --git a/docs/zh/14-reference/09-support-platform/index.md b/docs/zh/14-reference/09-support-platform/index.md index d396e4be1f..e676ab845a 100644 --- a/docs/zh/14-reference/09-support-platform/index.md +++ b/docs/zh/14-reference/09-support-platform/index.md @@ -5,18 +5,11 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" ## TDengine 服务端支持的平台列表 -| | **CentOS 7/8** | **Ubuntu 16/18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **华为 EulerOS** | -| ------------ | -------------- | ------------------- | --------------- | ------------ | ----------------- | ---------------- | ---------------- | -| X64 | ● | ● | | ○ | ● | ● | ● | -| 龙芯 MIPS64 | | | ● | | | | | -| 鲲鹏 ARM64 | | ○ | ○ | | ● | | | -| 申威 Alpha64 | | | ○ | ● | | | | -| 飞腾 ARM64 | | ○ 优麒麟 | | | | | | -| 海光 X64 | ● | ● | ● | ○ | ● | ● | | -| 瑞芯微 ARM64 | | | ○ | | | | | -| 全志 ARM64 | | | ○ | | | | | -| 炬力 ARM64 | | | ○ | | | | | -| 华为云 ARM64 | | | | | | | ● | +| | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **华为 EulerOS** | +| ------------ | ----------------- | ---------------- | ---------------- | --------------- | ------------ | ----------------- | ---------------- | ---------------- | +| X64 | ● | ● | ● | | ● | ● | ● | | +| 树莓派 ARM64 | | | | ● | | | | | +| 华为云 ARM64 | | | | | | | | ● | 注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。 diff --git a/docs/zh/14-reference/12-directory.md b/docs/zh/14-reference/12-directory.md index 0caf7e03c3..262eb99fa5 100644 --- a/docs/zh/14-reference/12-directory.md +++ b/docs/zh/14-reference/12-directory.md @@ -25,7 +25,6 @@ TDengine 的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 - _taosBenchmark_：TDengine 测试工具 - _remove.sh_：卸载 TDengine 的脚本，请谨慎执行，链接到/usr/bin 目录下的**rmtaos**命令。会删除 TDengine 的安装目录/usr/local/taos，但会保留/etc/taos、/var/lib/taos、/var/log/taos - _taosadapter_: 提供 RESTful 服务和接受其他多种软件写入请求的服务端可执行文件 -- _tarbitrator_: 提供双节点集群部署的仲裁功能 - _TDinsight.sh_：用于下载 TDinsight 并安装的脚本 - _set_core.sh_：用于方便调试设置系统生成 core dump 文件的脚本 - _taosd-dump-cfg.gdb_：用于方便调试 taosd 的 gdb 执行脚本。 From d3b2bcb61227e4665404a3286cd9f75c60947508 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 10 Aug 2022 19:39:01 +0800 Subject: [PATCH 135/165] doc: change 3.0 doc --- tools/shell/src/shellArguments.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 88ef46c5d6..818cc5425b 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -38,7 +38,7 @@ #define SHELL_STARTUP "Check the details of the service status." #define SHELL_WIDTH "Set the default binary display width, default is 30." #define SHELL_NET_ROLE "Net role when network connectivity test, options: client|server." -#define SHELL_PKG_LEN "Packet length used for net test, default is 1024 bytes." +#define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes." #define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." #define SHELL_VERSION "Print program version." #define SHELL_EMAIL "" @@ -62,7 +62,7 @@ void shellPrintHelp() { printf("%s%s%s%s\r\n", indent, "-f,", indent, SHELL_FILE); printf("%s%s%s%s\r\n", indent, "-h,", indent, SHELL_HOST); printf("%s%s%s%s\r\n", indent, "-k,", indent, SHELL_CHECK); - printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKG_LEN); + printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN); printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE); printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM); printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD); @@ -105,11 +105,11 @@ static struct argp_option shellOptions[] = { {"startup", 't', 0, 0, SHELL_STARTUP}, {"display-width", 'w', "WIDTH", 0, SHELL_WIDTH}, {"netrole", 'n', "NETROLE", 0, SHELL_NET_ROLE}, - {"pktlen", 'l', "PKTLEN", 0, SHELL_PKG_LEN}, + {"pktlen", 'l', "PKTLEN", 0, SHELL_PKT_LEN}, #ifdef WEBSOCKET {"dsn", 'E', "DSN", 0, SHELL_DSN}, {"restful", 'R', 0, 0, SHELL_REST}, - {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, + {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, {0}, @@ -228,7 +228,7 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { SShellArgs *pArgs = &shell.args; for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0) { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) { shellParseSingleOpt('?', NULL); return 0; } From ebf8755d9da1294ba1280295643d99903139aeea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 19:58:23 +0800 Subject: [PATCH 136/165] fix(query): remove invalid sma filter. --- source/libs/executor/src/scanoperator.c | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a211542de1..e59125a9bc 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -319,6 +319,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); return TSDB_CODE_SUCCESS; } else { + qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); *status = FUNC_DATA_REQUIRED_DATA_LOAD; } } @@ -326,20 +327,19 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD); // try to filter data block according to sma info - if (pTableScanInfo->pFilterNode != NULL) { - if (!loadSMA) { - doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); - } + if (pTableScanInfo->pFilterNode != NULL && (!loadSMA)) { + bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); + if (success) { + size_t size = taosArrayGetSize(pBlock->pDataBlock); + bool keep = doFilterByBlockSMA(pTableScanInfo->pFilterNode, pBlock->pBlockAgg, size, pBlockInfo->rows); + if (!keep) { + qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + pCost->filterOutBlocks += 1; + (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - bool keep = doFilterByBlockSMA(pTableScanInfo->pFilterNode, pBlock->pBlockAgg, taosArrayGetSize(pBlock->pDataBlock), - pBlockInfo->rows); - if (!keep) { - qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->filterOutBlocks += 1; - (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; + } } } From d8455841e01645750def9fff5bff6166233ff626 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 20:12:32 +0800 Subject: [PATCH 137/165] refactor: do some internal refactor. --- include/libs/function/function.h | 16 +- source/libs/executor/src/scanoperator.c | 75 - .../inc/{taggfunction.h => tfunctionInt.h} | 17 +- source/libs/function/src/builtinsimpl.c | 10 +- source/libs/function/src/taggfunction.c | 3910 ----------------- source/libs/function/src/texpr.c | 67 - source/libs/function/src/tfunctionInt.c | 86 + 7 files changed, 93 insertions(+), 4088 deletions(-) rename source/libs/function/inc/{taggfunction.h => tfunctionInt.h} (76%) delete mode 100644 source/libs/function/src/taggfunction.c delete mode 100644 source/libs/function/src/texpr.c create mode 100644 source/libs/function/src/tfunctionInt.c diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 72732ee198..e708a2c42d 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -54,10 +54,6 @@ typedef struct SFuncExecFuncs { FExecCombine combine; } SFuncExecFuncs; -typedef struct SFileBlockInfo { - int32_t numBlocksOfStep; -} SFileBlockInfo; - #define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results #define TOP_BOTTOM_QUERY_LIMIT 100 @@ -171,8 +167,6 @@ typedef struct tExprNode { }; } tExprNode; -void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); - struct SScalarParam { bool colAlloced; SColumnInfoData *columnData; @@ -182,14 +176,10 @@ struct SScalarParam { int32_t numOfRows; }; -int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, - bool isSuperTable); - -void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num); -void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); +void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock); -bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); -bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); +bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); +bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); typedef struct SPoint { int64_t key; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e59125a9bc..01a99fabee 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2458,81 +2458,6 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; -#if 0 - int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; - - STagScanInfo *pInfo = pOperator->info; - SSDataBlock *pRes = pInfo->pRes; - - int32_t count = 0; - SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); - - int32_t functionId = getExprFunctionId(&pOperator->exprSupp.pExprInfo[0]); - if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id - assert(pQueryAttr->numOfOutput == 1); - - SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; - int32_t rsize = pExprInfo->base.resSchema.bytes; - - count = 0; - - int16_t bytes = pExprInfo->base.resSchema.bytes; - int16_t type = pExprInfo->base.resSchema.type; - - for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) { - bytes = pQueryAttr->tagColList[i].bytes; - type = pQueryAttr->tagColList[i].type; - break; - } - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - STableQueryInfo *item = taosArrayGetP(pa, i); - - char *output = pColInfo->pData + count * rsize; - varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); - - output = varDataVal(output); - STableId* id = TSDB_TABLEID(item->pTable); - - *(int16_t *)output = 0; - output += sizeof(int16_t); - - *(int64_t *)output = id->uid; // memory align problem, todo serialize - output += sizeof(id->uid); - - *(int32_t *)output = id->tid; - output += sizeof(id->tid); - - *(int32_t *)output = pQueryAttr->vgId; - output += sizeof(pQueryAttr->vgId); - - char* data = NULL; - if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes); - } - - doSetTagValueToResultBuf(output, data, type, bytes); - count += 1; - } - - //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); - } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - *(int64_t*)pColInfo->pData = pInfo->totalTables; - count = 1; - - pOperator->status = OP_EXEC_DONE; - //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); - } else { // return only the tags|table name etc. -#endif - STagScanInfo* pInfo = pOperator->info; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; SSDataBlock* pRes = pInfo->pRes; diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/tfunctionInt.h similarity index 76% rename from source/libs/function/inc/taggfunction.h rename to source/libs/function/inc/tfunctionInt.h index 669c2635b5..8f6cbc977e 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/tfunctionInt.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TAGGFUNCTION_H -#define TDENGINE_TAGGFUNCTION_H +#ifndef TDENGINE_TFUNCTIONINT_H +#define TDENGINE_TFUNCTIONINT_H #ifdef __cplusplus extern "C" { @@ -28,17 +28,6 @@ extern "C" { #include "function.h" #include "tudf.h" -#define AVG_FUNCTION_INTER_BUFFER_SIZE 50 - -#define DATA_SET_FLAG ',' // to denote the output area has data, not null value -#define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) - -typedef struct SInterpInfoDetail { - TSKEY ts; // interp specified timestamp - int8_t type; - int8_t primaryCol; -} SInterpInfoDetail; - bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval); /** @@ -57,4 +46,4 @@ static FORCE_INLINE void initResultRowEntry(SResultRowEntryInfo *pResInfo, int32 } #endif -#endif // TDENGINE_TAGGFUNCTION_H +#endif // TDENGINE_TFUNCTIONINT_H diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index eb5d05c540..3e6e534328 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -18,10 +18,10 @@ #include "function.h" #include "query.h" #include "querynodes.h" -#include "taggfunction.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" +#include "tfunctionInt.h" #include "tglobal.h" #include "thistogram.h" #include "tpercentile.h" @@ -312,14 +312,6 @@ typedef struct SGroupKeyInfo { #define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) #define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) -#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SqlFunctionCtx* __ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - __ctx->fpSet.process(__ctx); \ - } \ - } while (0); - #define DO_UPDATE_SUBSID_RES(ctx, ts) \ do { \ for (int32_t _i = 0; _i < (ctx)->subsidiaries.num; ++_i) { \ diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c deleted file mode 100644 index c8998b9d94..0000000000 --- a/source/libs/function/src/taggfunction.c +++ /dev/null @@ -1,3910 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "taosdef.h" -#include "tmsg.h" -#include "thash.h" -#include "ttypes.h" - -#include "function.h" -#include "taggfunction.h" -#include "tbuffer.h" -#include "tcompression.h" -#include "thistogram.h" -#include "tpercentile.h" -#include "ttszip.h" -#include "tdatablock.h" -#include "tudf.h" - -#define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) -#define GET_INPUT_DATA(x, y) ((char*) colDataGetData((x)->pInput, (y))) - -#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) -#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) - -#define GET_TRUE_DATA_TYPE() \ - int32_t type = 0; \ - if (pCtx->scanFlag == MERGE_STAGE) { \ - type = pCtx->resDataInfo.type; \ - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \ - } else { \ - type = pCtx->inputType; \ - } - -#define SET_VAL(ctx, numOfElem, res) \ - do { \ - if ((numOfElem) <= 0) { \ - break; \ - } \ - GET_RES_INFO(ctx)->numOfRes = (res); \ - } while (0) - -#define INC_INIT_VAL(ctx, res) (GET_RES_INFO(ctx)->numOfRes += (res)); - -#define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ - __ctx->tag.i = (ts); \ - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ - } \ - aggFunc[FUNCTION_TAG].addInput(__ctx); \ - } \ - } while (0) - -#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - aggFunc[FUNCTION_TAG].addInput(__ctx); \ - } \ - } while (0); - -void noop1(SqlFunctionCtx *UNUSED_PARAM(pCtx)) {} - -void doFinalizer(SqlFunctionCtx *pCtx) { cleanupResultRowEntry(GET_RES_INFO(pCtx)); } - -typedef struct tValuePair { - SVariant v; - int64_t timestamp; - char * pTags; // the corresponding tags of each record in the final result -} tValuePair; - -typedef struct SSpreadInfo { - double min; - double max; - int8_t hasResult; -} SSpreadInfo; - -typedef struct SSumInfo { - union { - int64_t isum; - uint64_t usum; - double dsum; - }; - int8_t hasResult; -} SSumInfo; - -// the attribute of hasResult is not needed since the num attribute would server as this purpose -typedef struct SAvgInfo { - double sum; - int64_t num; -} SAvgInfo; - -typedef struct SStddevInfo { - double avg; - int64_t num; - double res; - int8_t stage; -} SStddevInfo; - -typedef struct SStddevdstInfo { - int64_t num; - double res; -} SStddevdstInfo; - -typedef struct SFirstLastInfo { - int8_t hasResult; - TSKEY ts; -} SFirstLastInfo; - -typedef struct SFirstLastInfo SLastrowInfo; -typedef struct SPercentileInfo { - tMemBucket *pMemBucket; - int32_t stage; - double minval; - double maxval; - int64_t numOfElems; -} SPercentileInfo; - -typedef struct STopBotInfo { - int32_t num; - tValuePair **res; -} STopBotInfo; - -// leastsquares do not apply to super table -typedef struct SLeastsquaresInfo { - double mat[2][3]; - double startVal; - int64_t num; -} SLeastsquaresInfo; - -typedef struct SAPercentileInfo { - SHistogramInfo *pHisto; -} SAPercentileInfo; - -typedef struct STSCompInfo { - STSBuf *pTSBuf; -} STSCompInfo; - -typedef struct SRateInfo { - double correctionValue; - double firstValue; - TSKEY firstKey; - double lastValue; - TSKEY lastKey; - int8_t hasResult; // flag to denote has value - bool isIRate; // true for IRate functions, false for Rate functions -} SRateInfo; - -//typedef struct SDerivInfo { -// double prevValue; // previous value -// TSKEY prevTs; // previous timestamp -// bool ignoreNegative;// ignore the negative value -// int64_t tsWindow; // time window for derivative -// bool valueSet; // the value has been set already -//} SDerivInfo; - -typedef struct SResPair { - TSKEY key; - double avg; -} SResPair; - -void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { - pCell->initialized = false; -} - -int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock) { - int32_t maxRows = 0; - - for (int32_t j = 0; j < num; ++j) { -#if 0 - int32_t id = pCtx[j].functionId; - - /* - * ts, tag, tagprj function can not decide the output number of current query - * the number of output result is decided by main output - */ - if (id == FUNCTION_TS || id == FUNCTION_TAG || id == FUNCTION_TAGPRJ) { - continue; - } -#endif - SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); - if (pResInfo != NULL && maxRows < pResInfo->numOfRes) { - maxRows = pResInfo->numOfRes; - } - } - - assert(maxRows >= 0); - - blockDataEnsureCapacity(pResBlock, maxRows); - for(int32_t i = 0; i < num; ++i) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[i]); - if (pResInfo->numOfRes == 0) { - for(int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataAppend(pCol, j, NULL, true); // TODO add set null data api - } - } else { - for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataAppend(pCol, j, GET_ROWCELL_INTERBUF(pResInfo), false); - } - } - } - - pResBlock->info.rows = maxRows; - return maxRows; -} - -void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num) { - for (int32_t j = 0; j < num; ++j) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); - pResInfo->numOfRes = 0; - } -} - -bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { - assert(pEntry != NULL); - return pEntry->complete; -} - -bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry) { - return pEntry->initialized; -} -#if 0 -int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, - bool isSuperTable/*, SUdfInfo* pUdfInfo*/) { - if (!isValidDataType(dataType)) { -// qError("Illegal data type %d or data type length %d", dataType, dataBytes); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - - if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG_DUMMY || - functionId == FUNCTION_DIFF || functionId == FUNCTION_PRJ || functionId == FUNCTION_TAGPRJ || - functionId == FUNCTION_TAG || functionId == FUNCTION_INTERP) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - - if (functionId == FUNCTION_INTERP) { - pInfo->interBufSize = sizeof(SInterpInfoDetail); - } else { - pInfo->interBufSize = 0; - } - - return TSDB_CODE_SUCCESS; - } - - // (uid, tid) + VGID + TAGSIZE + VARSTR_HEADER_SIZE - if (functionId == FUNCTION_TID_TAG) { // todo use struct - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_BLKINFO) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = 16384; - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_COUNT) { - pInfo->type = TSDB_DATA_TYPE_BIGINT; - pInfo->bytes = sizeof(int64_t); - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_ARITHM) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_TS_COMP) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = 1; // this results is compressed ts data, only one byte - pInfo->interBufSize = POINTER_BYTES; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_DERIVATIVE) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); // this results is compressed ts data, only one byte - pInfo->interBufSize = sizeof(SDerivInfo); - return TSDB_CODE_SUCCESS; - } - - if (isSuperTable) { -// if (functionId < 0) { -// if (pUdfInfo->bufSize > 0) { -// pInfo->type = TSDB_DATA_TYPE_BINARY; -// pInfo->bytes = pUdfInfo->bufSize; -// pInfo->interBufSize = pInfo->bytes; -// } else { -// pInfo->type = pUdfInfo->resType; -// pInfo->bytes = pUdfInfo->resBytes; -// pInfo->interBufSize = pInfo->bytes; -// } -// -// return TSDB_CODE_SUCCESS; -// } - - if (functionId == FUNCTION_MIN || functionId == FUNCTION_MAX) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(dataBytes + DATA_SET_FLAG_SIZE); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_SUM) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SSumInfo); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_AVG) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SAvgInfo); - pInfo->interBufSize = pInfo->bytes; - return TSDB_CODE_SUCCESS; - - } else if (functionId >= FUNCTION_RATE && functionId <= FUNCTION_IRATE) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(SRateInfo); - pInfo->interBufSize = sizeof(SRateInfo); - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_SPREAD) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SSpreadInfo); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_APERCT) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo) + sizeof(SAPercentileInfo); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_LAST_ROW) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(sizeof(SLastrowInfo) + dataBytes); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_TWA) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(STwaInfo); - pInfo->interBufSize = pInfo->bytes; - return TSDB_CODE_SUCCESS; - } - } - - if (functionId == FUNCTION_SUM) { - if (IS_SIGNED_NUMERIC_TYPE(dataType)) { - pInfo->type = TSDB_DATA_TYPE_BIGINT; - } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { - pInfo->type = TSDB_DATA_TYPE_UBIGINT; - } else { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - } - - pInfo->bytes = sizeof(int64_t); - pInfo->interBufSize = sizeof(SSumInfo); - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_APERCT) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = - sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_TWA) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(STwaInfo); - return TSDB_CODE_SUCCESS; - } - -// if (functionId < 0) { -// pInfo->type = pUdfInfo->resType; -// pInfo->bytes = pUdfInfo->resBytes; -// -// if (pUdfInfo->bufSize > 0) { -// pInfo->interBufSize = pUdfInfo->bufSize; -// } else { -// pInfo->interBufSize = pInfo->bytes; -// } -// -// return TSDB_CODE_SUCCESS; -// } - - if (functionId == FUNCTION_AVG) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SAvgInfo); - } else if (functionId >= FUNCTION_RATE && functionId <= FUNCTION_IRATE) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SRateInfo); - } else if (functionId == FUNCTION_STDDEV) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SStddevInfo); - } else if (functionId == FUNCTION_MIN || functionId == FUNCTION_MAX) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - pInfo->interBufSize = dataBytes + DATA_SET_FLAG_SIZE; - } else if (functionId == FUNCTION_FIRST || functionId == FUNCTION_LAST) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - pInfo->interBufSize = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); - } else if (functionId == FUNCTION_SPREAD) { - pInfo->type = (int16_t)TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SSpreadInfo); - } else if (functionId == FUNCTION_PERCT) { - pInfo->type = (int16_t)TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = (int16_t)sizeof(double); - pInfo->interBufSize = (int16_t)sizeof(SPercentileInfo); - } else if (functionId == FUNCTION_LEASTSQR) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = TMAX(AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string - pInfo->interBufSize = pInfo->bytes; - } else if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); - pInfo->interBufSize = pInfo->bytes; - } else if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - - size_t size = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; - - // the output column may be larger than sizeof(STopBotInfo) - pInfo->interBufSize = (int32_t)size; - } else if (functionId == FUNCTION_LAST_ROW) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - pInfo->interBufSize = dataBytes; - } else if (functionId == FUNCTION_STDDEV_DST) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SStddevdstInfo); - pInfo->interBufSize = (pInfo->bytes); - - } else { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} -#endif - -static bool function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (pResultInfo->initialized) { - return false; - } - - memset(pCtx->pOutput, 0, (size_t)pCtx->resDataInfo.bytes); - initResultRowEntry(pResultInfo, pCtx->resDataInfo.interBufSize); - return true; -} -#if 0 -/** - * in handling the stable query, function_finalizer is called after the secondary - * merge being completed, during the first merge procedure, which is executed at the - * vnode side, the finalize will never be called. - * - * @param pCtx - */ -static void function_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); -// if (pResInfo->hasResult != DATA_SET_FLAG) { // TODO set the correct null value -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// } - - doFinalizer(pCtx); -} - -/** - * 1. If the column value for filter exists, we need to load the SFields, which serves - * as the pre-filter to decide if the actual data block is required or not. - * 2. If it queries on the non-primary timestamp column, SFields is also required to get the not-null value. - * - * @param colId - * @param filterCols - * @return - */ -int32_t countRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - return BLK_DATA_NOT_LOAD; - } else { - return BLK_DATA_SMA_LOAD; - } -} - -int32_t noDataRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_NOT_LOAD; -} -#define LIST_ADD_N_DOUBLE_FLOAT(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - SET_DOUBLE_VAL(&(x) , GET_DOUBLE_VAL(&(x)) + GET_FLOAT_VAL(&(d)[i])); \ - (numOfElem)++; \ - } \ - } while(0) -#define LIST_ADD_N_DOUBLE(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - SET_DOUBLE_VAL(&(x) , (x) + (d)[i]); \ - (numOfElem)++; \ - } \ - } while(0) - -#define LIST_ADD_N(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - (x) += (d)[i]; \ - (numOfElem)++; \ - } \ - } while(0) - -#define UPDATE_DATA(ctx, left, right, num, sign, k) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ - DO_UPDATE_TAG_COLUMNS(ctx, k); \ - (num) += 1; \ - } \ - } while (0) - -#define DUPATE_DATA_WITHOUT_TS(ctx, left, right, num, sign) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ - DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx); \ - (num) += 1; \ - } \ - } while (0) - -#define LOOPCHECK_N(val, list, ctx, tsdbType, sign, num) \ - for (int32_t i = 0; i < ((ctx)->size); ++i) { \ - if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ - continue; \ - } \ - TSKEY key = (ctx)->ptsList != NULL? GET_TS_DATA(ctx, i):0; \ - UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \ - } - -#define TYPED_LOOPCHECK_N(type, data, list, ctx, tsdbType, sign, notNullElems) \ - do { \ - type *_data = (type *)data; \ - type *_list = (type *)list; \ - LOOPCHECK_N(*_data, _list, ctx, tsdbType, sign, notNullElems); \ - } while (0) - -static int32_t statisRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_SMA_LOAD; -} - -static int32_t dataBlockRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_DATA_LOAD; -} - -// todo: if column in current data block are null, opt for this case -static int32_t firstFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order == TSDB_ORDER_DESC) { - return BLK_DATA_NOT_LOAD; - } - - // no result for first query, data block is required - if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { - return BLK_DATA_DATA_LOAD; - } else { - return BLK_DATA_NOT_LOAD; - } -} - -static int32_t lastFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { -// if (pCtx->order != pCtx->param[0].param.i) { -// return BLK_DATA_NOT_LOAD; -// } - - if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { - return BLK_DATA_DATA_LOAD; - } else { - return BLK_DATA_NOT_LOAD; - } -} - -static int32_t firstDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order == TSDB_ORDER_DESC) { - return BLK_DATA_NOT_LOAD; - } - - // not initialized yet, it is the first block, load it. - if (pCtx->pOutput == NULL) { - return BLK_DATA_DATA_LOAD; - } - - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is - // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG) { - return BLK_DATA_DATA_LOAD; - } else { // data in current block is not earlier than current result - return (pInfo->ts <= w->skey) ? BLK_DATA_NOT_LOAD : BLK_DATA_DATA_LOAD; - } -} - -static int32_t lastDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { -// if (pCtx->order != pCtx->param[0].param.i) { -// return BLK_DATA_NOT_LOAD; -// } - - // not initialized yet, it is the first block, load it. - if (pCtx->pOutput == NULL) { - return BLK_DATA_DATA_LOAD; - } - - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is - // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG) { - return BLK_DATA_DATA_LOAD; - } else { - return (pInfo->ts > w->ekey) ? BLK_DATA_NOT_LOAD : BLK_DATA_DATA_LOAD; - } -} - -////////////////////////////////////////////////////////////////////////////////////////////// -/* - * The intermediate result of average is kept in the interResultBuf. - * For super table query, once the avg_function/avg_function_f is finished, copy the intermediate - * result into output buffer. - */ -static void avg_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - // NOTE: keep the intermediate result into the interResultBuf - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - double *pVal = &pAvgInfo->sum; - - if (pCtx->isAggSet) { // Pre-aggregation - notNullElems = pCtx->size - pCtx->agg.numOfNull; - assert(notNullElems >= 0); - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += pCtx->agg.sum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += (uint64_t) pCtx->agg.sum; - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->agg.sum)); - } - } else { - void *pData = GET_INPUT_DATA_LIST(pCtx); - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(*pVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(*pVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(*pVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_ADD_N_DOUBLE(*pVal, pCtx, pData, double, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_ADD_N_DOUBLE_FLOAT(*pVal, pCtx, pData, float, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); - } - } - - if (!pCtx->hasNull) { - assert(notNullElems == pCtx->size); - } - - SET_VAL(pCtx, notNullElems, 1); - pAvgInfo->num += notNullElems; - - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); - } -} - -static void avg_func_merge(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - double *sum = (double*) pCtx->pOutput; - char *input = GET_INPUT_DATA_LIST(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SAvgInfo *pInput = (SAvgInfo *)input; - if (pInput->num == 0) { // current input is null - continue; - } - - SET_DOUBLE_VAL(sum, *sum + pInput->sum); - - // keep the number of data into the temp buffer - *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num; - } -} - -/* - * the average value is calculated in finalize routine, since current routine does not know the exact number of points - */ -static void avg_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->scanFlag == MERGE_STAGE) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput,(*(double *)pCtx->pOutput) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo)); - } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY - assert(IS_NUMERIC_TYPE(pCtx->inputType)); - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pAvgInfo->num == 0) { // all data are NULL or empty table - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pAvgInfo->sum / pAvgInfo->num); - } - - // cannot set the numOfIteratedElems again since it is set during previous iteration - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////////////////// - -static bool min_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; // not initialized since it has been initialized - } - - GET_TRUE_DATA_TYPE(); - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->pOutput) = INT8_MAX; - break; - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *) pCtx->pOutput = UINT8_MAX; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->pOutput) = INT16_MAX; - break; - case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->pOutput) = UINT16_MAX; - break; - case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->pOutput) = INT32_MAX; - break; - case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->pOutput) = UINT32_MAX; - break; - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->pOutput) = INT64_MAX; - break; - case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->pOutput) = UINT64_MAX; - break; - case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->pOutput) = FLT_MAX; - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)pCtx->pOutput), DBL_MAX); - break; - default: - assert(0); -// qError("illegal data type:%d in min/max query", pCtx->inputType); - } - - return true; -} - -static bool max_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; // not initialized since it has been initialized - } - - GET_TRUE_DATA_TYPE(); - - switch (type) { - case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->pOutput) = INT32_MIN; - break; - case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->pOutput) = -FLT_MAX; - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)pCtx->pOutput), -DBL_MAX); - break; - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->pOutput) = INT64_MIN; - break; - case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->pOutput) = INT16_MIN; - break; - case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->pOutput) = INT8_MIN; - break; - case TSDB_DATA_TYPE_UTINYINT: - *((uint8_t *)pCtx->pOutput) = 0; - break; - default: - assert(0); -// qError("illegal data type:%d in min/max query", pCtx->inputType); - } - - return true; -} - -/* - * the output result of min/max function is the final output buffer, not the intermediate result buffer - */ -static int32_t minmax_merge_impl(SqlFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { - int32_t notNullElems = 0; -#if 0 - GET_TRUE_DATA_TYPE(); - assert(pCtx->stableQuery); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *input = GET_INPUT_DATA(pCtx, i); - if (input[bytes] != DATA_SET_FLAG) { - continue; - } - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t v = GET_INT8_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int8_t *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t v = GET_INT16_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int16_t *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t v = GET_INT32_VAL(input); - if ((*(int32_t *)output < v) ^ isMin) { - *(int32_t *)output = v; - - for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - SqlFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; - aggFunc[FUNCTION_TAG].addInput(__ctx); - } - - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float v = GET_FLOAT_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(float *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double v = GET_DOUBLE_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(double *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t v = GET_INT64_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int64_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t v = GET_UINT8_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint8_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t v = GET_UINT16_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint16_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UINT: { - uint32_t v = GET_UINT32_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint32_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t v = GET_UINT64_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint64_t *)output, v, notNullElems, isMin); - break; - } - - default: - break; - } - } -#endif - - return notNullElems; -} - -static void min_func_merge(SqlFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 1); - - SET_VAL(pCtx, notNullElems, 1); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void max_func_merge(SqlFunctionCtx *pCtx) { - int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 0); - - SET_VAL(pCtx, numOfElem, 1); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - if (numOfElem > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -#define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, _type, num) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], (_type))) { \ - continue; \ - } \ - (num) += 1; \ - (r) += TPOW2(((type *)d)[i] - (delta)); \ - } - -static void stddev_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->scanFlag == REPEAT_SCAN && pStd->stage == 0) { - pStd->stage++; - avg_finalizer(pCtx); - - pResInfo->initialized = true; // set it initialized to avoid re-initialization - - // save average value into tmpBuf, for second stage scan - SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); - - pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); - assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); - } - - if (pStd->stage == 0) { - // the first stage is to calculate average value - avg_function(pCtx); - } else if (pStd->num > 0) { - // the second stage to calculate standard deviation - // if pStd->num == 0, there are no numbers in the first round check. No need to do the second round - double *retVal = &pStd->res; - double avg = pStd->avg; - - void *pData = GET_INPUT_DATA_LIST(pCtx); - int32_t num = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { - continue; - } - num += 1; - *retVal += TPOW2(((int32_t *)pData)[i] - avg); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - LOOP_STDDEV_IMPL(uint8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UINT: { - LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - default: - assert(0); -// qError("stddev function not support data type:%d", pCtx->inputType); - } - - SET_VAL(pCtx, 1, 1); - } -} - -static void stddev_finalizer(SqlFunctionCtx *pCtx) { - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - if (pStd->num <= 0) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else { - double *retValue = (double *)pCtx->pOutput; - SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num)); - SET_VAL(pCtx, 1, 1); - } - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////// -int32_t tsCompare(const void* p1, const void* p2) { - TSKEY k = *(TSKEY*)p1; - SResPair* pair = (SResPair*)p2; - - if (k == pair->key) { - return 0; - } else { - return k < pair->key? -1:1; - } -} - -////////////////////////////////////////////////////////////////////////////////////// -static bool first_last_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - // used to keep the timestamp for comparison -// pCtx->param[1].param.nType = 0; -// pCtx->param[1].param.i = 0; - - return true; -} - -// todo opt for null block -static void first_function(SqlFunctionCtx *pCtx) { - if (pCtx->order == TSDB_ORDER_DESC) { - return; - } - - int32_t notNullElems = 0; - - // handle the null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - if (pCtx->ptsList != NULL) { - TSKEY k = GET_TS_DATA(pCtx, i); -// DO_UPDATE_TAG_COLUMNS(pCtx, k); - } - - SResultRowEntryInfo *pInfo = GET_RES_INFO(pCtx); -// pInfo->hasResult = DATA_SET_FLAG; - pInfo->complete = true; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void first_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = GET_TS_LIST(pCtx); - - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - - if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) { - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); - } -} - -/* - * format of intermediate result: "timestamp,value" need to compare the timestamp in the first part (before the comma) - * to decide if the value is earlier than current intermediate result - */ -static void first_dist_function(SqlFunctionCtx *pCtx) { - /* - * do not to check data in the following cases: - * 1. data block that are not loaded - * 2. scan data files in desc order - */ - if (pCtx->order == TSDB_ORDER_DESC) { - return; - } - - int32_t notNullElems = 0; - - // find the first not null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - first_data_assign_impl(pCtx, data, i); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void first_dist_func_merge(SqlFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - char * pData = GET_INPUT_DATA_LIST(pCtx); - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->resDataInfo.bytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - // The param[1] is used to keep the initial value of max ts value -// if (pCtx->param[1].param.nType != pCtx->resDataInfo.type || pCtx->param[1].param.i > pInput->ts) { -// memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); -// pCtx->param[1].param.i = pInput->ts; -// pCtx->param[1].param.nType = pCtx->resDataInfo.type; -// -//// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); -// } - - SET_VAL(pCtx, 1, 1); -// GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -////////////////////////////////////////////////////////////////////////////////////////// -/* - * last function: - * 1. since the last block may be all null value, so, we simply access the last block is not valid - * each block need to be checked. - * 2. If numOfNull == pBlock->numOfBlocks, the whole block is empty. Otherwise, there is at - * least one data in this block that is not null.(TODO opt for this case) - */ -static void last_function(SqlFunctionCtx *pCtx) { -// if (pCtx->order != pCtx->param[0].param.i) { -// return; -// } - - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - if (pCtx->order == TSDB_ORDER_DESC) { - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { - continue; - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - - TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); - - //pResInfo->hasResult = DATA_SET_FLAG; - pResInfo->complete = true; // set query completed on this column - notNullElems++; - break; - } - } else { // ascending order - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { - continue; - } - - TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; - - char* buf = GET_ROWCELL_INTERBUF(pResInfo); -// if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { -// //pResInfo->hasResult = DATA_SET_FLAG; -// memcpy(pCtx->pOutput, data, pCtx->inputBytes); -// -// *(TSKEY*)buf = ts; -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); -// } - - notNullElems++; - break; - } - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void last_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = GET_TS_LIST(pCtx); - - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - - if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) { -#if defined(_DEBUG_VIEW) - qDebug("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData); -#endif - - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); - } -} - -static void last_dist_function(SqlFunctionCtx *pCtx) { - /* - * 1. for scan data is not the required order - * 2. for data blocks that are not loaded, no need to check data - */ -// if (pCtx->order != pCtx->param[0].param.i) { -// return; -// } - - int32_t notNullElems = 0; - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - if (!pCtx->requireNull) { - continue; - } - } - - last_data_assign_impl(pCtx, data, i); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -/* - * in the secondary merge(local reduce), the output is limited by the - * final output size, so the main difference between last_dist_func_merge and second_merge - * is: the output data format in computing - */ -static void last_dist_func_merge(SqlFunctionCtx *pCtx) { - char *pData = GET_INPUT_DATA_LIST(pCtx); - - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->resDataInfo.bytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - /* - * param[1] used to keep the corresponding timestamp to decide if current result is - * the true last result - */ - if (pCtx->param[1].param.nType != pCtx->resDataInfo.type || pCtx->param[1].param.i < pInput->ts) { - memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); - pCtx->param[1].param.i = pInput->ts; - pCtx->param[1].param.nType = pCtx->resDataInfo.type; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } - - SET_VAL(pCtx, 1, 1); -// GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -////////////////////////////////////////////////////////////////////////////////// -/* - * NOTE: last_row does not use the interResultBuf to keep the result - */ -static void last_row_function(SqlFunctionCtx *pCtx) { - assert(pCtx->size >= 1); - char *pData = GET_INPUT_DATA_LIST(pCtx); - - // assign the last element in current data block - assignVal(pCtx->pOutput, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - // set the result to final result buffer in case of super table query - if (pCtx->stableQuery) { - SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->pOutput + pCtx->inputBytes); - pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); - pInfo1->hasResult = DATA_SET_FLAG; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); - } else { - TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1); -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); - } - - SET_VAL(pCtx, pCtx->size, 1); -} - -static void last_row_finalizer(SqlFunctionCtx *pCtx) { - // do nothing at the first stage - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); -// if (pResInfo->hasResult != DATA_SET_FLAG) { -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// return; -// } - - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////// -static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int64_t tsKey, char *pTags, - SSubsidiaryResInfo *pTagInfo, int16_t stage) { - dst->v.nType = type; - dst->v.i = *(int64_t *)val; - dst->timestamp = tsKey; - - int32_t size = 0; - if (stage == MERGE_STAGE) { -// memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); - } else { // the tags are dumped from the ctx tag fields -// for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { -// SqlFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; -// if (ctx->functionId == FUNCTION_TS_DUMMY) { -// ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; -// ctx->tag.i = tsKey; -// } -// -// taosVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); -// size += pTagInfo->pTagCtxList[i]->resDataInfo.bytes; -// } - } -} - -#define VALUEPAIRASSIGN(dst, src, __l) \ - do { \ - (dst)->timestamp = (src)->timestamp; \ - (dst)->v = (src)->v; \ - memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ - } while (0) - -static int32_t topBotComparFn(const void *p1, const void *p2, const void *param) -{ - uint16_t type = *(uint16_t *) param; - tValuePair *val1 = *(tValuePair **) p1; - tValuePair *val2 = *(tValuePair **) p2; - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - if (val1->v.i == val2->v.i) { - return 0; - } - - return (val1->v.i > val2->v.i) ? 1 : -1; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - if (val1->v.u == val2->v.u) { - return 0; - } - - return (val1->v.u > val2->v.u) ? 1 : -1; - } - - if (val1->v.d == val2->v.d) { - return 0; - } - - return (val1->v.d > val2->v.d) ? 1 : -1; -} - -static void topBotSwapFn(void *dst, void *src, const void *param) -{ - char tag[32768]; - tValuePair temp; - uint16_t tagLen = *(uint16_t *) param; - tValuePair *vdst = *(tValuePair **) dst; - tValuePair *vsrc = *(tValuePair **) src; - - memset(tag, 0, sizeof(tag)); - temp.pTags = tag; - - VALUEPAIRASSIGN(&temp, vdst, tagLen); - VALUEPAIRASSIGN(vdst, vsrc, tagLen); - VALUEPAIRASSIGN(vsrc, &temp, tagLen); -} - -static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SSubsidiaryResInfo *pTagInfo, char *pTags, int16_t stage) { - SVariant val = {0}; - taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); - - tValuePair **pList = pInfo->res; - assert(pList != NULL); - - if (pInfo->num < maxLen) { - valuePairAssign(pList[pInfo->num], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - -// taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); - - pInfo->num++; - } else { - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pList[0]->v.i) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pList[0]->v.u) || - (IS_FLOAT_TYPE(type) && val.d > pList[0]->v.d)) { - valuePairAssign(pList[0], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); -// taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); - } - } -} - -static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SSubsidiaryResInfo *pTagInfo, char *pTags, int16_t stage) { - SVariant val = {0}; - taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); - - tValuePair **pList = pInfo->res; - assert(pList != NULL); - - if (pInfo->num < maxLen) { - valuePairAssign(pList[pInfo->num], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - -// taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); - - pInfo->num++; - } else { - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i < pList[0]->v.i) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u < pList[0]->v.u) || - (IS_FLOAT_TYPE(type) && val.d < pList[0]->v.d)) { - valuePairAssign(pList[0], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); -// taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); - } - } -} - -static int32_t resAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = *(tValuePair **)pLeft; - tValuePair *pRightElem = *(tValuePair **)pRight; - - if (pLeftElem->timestamp == pRightElem->timestamp) { - return 0; - } else { - return pLeftElem->timestamp > pRightElem->timestamp ? 1 : -1; - } -} - -static int32_t resDescComparFn(const void *pLeft, const void *pRight) { return -resAscComparFn(pLeft, pRight); } - -static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = *(tValuePair **)pLeft; - tValuePair *pRightElem = *(tValuePair **)pRight; - - if (IS_FLOAT_TYPE(pLeftElem->v.nType)) { - if (pLeftElem->v.d == pRightElem->v.d) { - return 0; - } else { - return pLeftElem->v.d > pRightElem->v.d ? 1 : -1; - } - } else if (IS_SIGNED_NUMERIC_TYPE(pLeftElem->v.nType)){ - if (pLeftElem->v.i == pRightElem->v.i) { - return 0; - } else { - return pLeftElem->v.i > pRightElem->v.i ? 1 : -1; - } - } else { - if (pLeftElem->v.u == pRightElem->v.u) { - return 0; - } else { - return pLeftElem->v.u > pRightElem->v.u ? 1 : -1; - } - } -} - -static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } - -static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); - - tValuePair **tvp = pRes->res; - - int32_t step = QUERY_ASC_FORWARD_STEP; - int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes); - - switch (type) { - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - int32_t *output = (int32_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int32_t)tvp[i]->v.i; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - int64_t *output = (int64_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i]->v.i; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *output = (double *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - SET_DOUBLE_VAL(output, tvp[i]->v.d); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *output = (float *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (float)tvp[i]->v.d; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *output = (int16_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int16_t)tvp[i]->v.i; - } - break; - } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - int8_t *output = (int8_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int8_t)tvp[i]->v.i; - } - break; - } - default: { -// qError("top/bottom function not support data type:%d", pCtx->inputType); - return; - } - } - - // set the output timestamp of each record. -// TSKEY *output = pCtx->pTsOutput; -// for (int32_t i = 0; i < len; ++i, output += step) { -// *output = tvp[i]->timestamp; -// } - - // set the corresponding tag data for each record - // todo check malloc failure -// char **pData = taosMemoryCalloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); -// for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { -// pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; -// } - -// for (int32_t i = 0; i < len; ++i, output += step) { -// int16_t offset = 0; -// for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { -// memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes); -// offset += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; -// pData[j] += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; -// } -// } - -// taosMemoryFreeClear(pData); -} - -/* - * Parameters values: - * 1. param[0]: maximum allowable results - * 2. param[1]: order by type (time or value) - * 3. param[2]: asc/desc order - * - * top/bottom use the intermediate result buffer to keep the intermediate result - */ -static STopBotInfo *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - // only the first_stage_merge is directly written data into final output buffer - if (pCtx->stableQuery && pCtx->scanFlag != MERGE_STAGE) { - return (STopBotInfo*) pCtx->pOutput; - } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer - return GET_ROWCELL_INTERBUF(pResInfo); - } -} - - -/* - * keep the intermediate results during scan data blocks in the format of: - * +-----------------------------------+-------------one value pair-----------+------------next value pair-----------+ - * |-------------pointer area----------|----ts---+-----+-----n tags-----------|----ts---+-----+-----n tags-----------| - * +..[Value Pointer1][Value Pointer2].|timestamp|value|tags1|tags2|....|tagsn|timestamp|value|tags1|tags2|....|tagsn+ - */ -static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SqlFunctionCtx *pCtx) { - char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); - pTopBotInfo->res = (tValuePair**) tmp; -// tmp += POINTER_BYTES * pCtx->param[0].param.i; - -// size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; - -// for (int32_t i = 0; i < pCtx->param[0].param.i; ++i) { -// pTopBotInfo->res[i] = (tValuePair*) tmp; -// pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); -// tmp += size; -// } -} - -bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo == NULL) { - return true; - } - - STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx); - - // required number of results are not reached, continue load data block -// if (pTopBotInfo->num < pCtx->param[0].param.i) { -// return true; -// } - -// if ((void *)pTopBotInfo->res[0] != (void *)((char *)pTopBotInfo + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { -// buildTopBotStruct(pTopBotInfo, pCtx); -// } - - tValuePair **pRes = (tValuePair**) pTopBotInfo->res; - - if (pCtx->functionId == FUNCTION_TOP) { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(maxval) > pRes[0]->v.d; - case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(maxval) > pRes[0]->v.d; - default: - return true; - } - } else { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(minval) < pRes[0]->v.d; - case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(minval) < pRes[0]->v.d; - default: - return true; - } - } -} - -static bool top_bottom_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - STopBotInfo *pInfo = getTopBotOutputInfo(pCtx); - buildTopBotStruct(pInfo, pCtx); - return true; -} - -static void top_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - assert(pRes->num >= 0); - -// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { -// buildTopBotStruct(pRes, pCtx); -// } - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems++; - - // NOTE: Set the default timestamp if it is missing [todo refactor] - TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; -// do_top_function_add(pRes, (int32_t)pCtx->param[0].param.i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - // treat the result as only one result - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void top_func_merge(SqlFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); - - // construct the input data struct from binary data - buildTopBotStruct(pInput, pCtx); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - // the intermediate result is binary, we only use the output data type - for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->resDataInfo.type; -// do_top_function_add(pOutput, (int32_t)pCtx->param[0].param.i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, -// type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->scanFlag); - } - - SET_VAL(pCtx, pInput->num, pOutput->num); - - if (pOutput->num > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void bottom_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - -// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { -// buildTopBotStruct(pRes, pCtx); -// } - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems++; - // NOTE: Set the default timestamp if it is missing [todo refactor] - TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; -// do_bottom_function_add(pRes, (int32_t)pCtx->param[0].param.i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - // treat the result as only one result - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void bottom_func_merge(SqlFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); - - // construct the input data struct from binary data - buildTopBotStruct(pInput, pCtx); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - // the intermediate result is binary, we only use the output data type - for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->resDataInfo.type; -// do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].param.i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, -// &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->scanFlag); - } - - SET_VAL(pCtx, pInput->num, pOutput->num); - - if (pOutput->num > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void top_bottom_func_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - // data in temporary list is less than the required number of results, not enough qualified number of results - STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); - if (pRes->num == 0) { // no result -// assert(pResInfo->hasResult != DATA_SET_FLAG); - // TODO: - } - - GET_RES_INFO(pCtx)->numOfRes = pRes->num; - tValuePair **tvp = pRes->res; - - // user specify the order of output by sort the result according to timestamp - if (pCtx->param[1].param.i == PRIMARYKEY_TIMESTAMP_COL_ID) { - __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; - taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } else /*if (pCtx->param[1].param.i > PRIMARYKEY_TIMESTAMP_COL_ID)*/ { - __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; - taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } - - GET_TRUE_DATA_TYPE(); - copyTopBotRes(pCtx, type); - - doFinalizer(pCtx); -} - -/////////////////////////////////////////////////////////////////////////////////////////////// -static bool percentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - // in the first round, get the min-max value of all involved data - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); - SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX); - SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); - pInfo->numOfElems = 0; - - return true; -} - -static void percentile_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->scanFlag == REPEAT_SCAN && pInfo->stage == 0) { - pInfo->stage += 1; - - // all data are null, set it completed - if (pInfo->numOfElems == 0) { - pResInfo->complete = true; - - return; - } else { - pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); - } - } - - // the first stage, only acquire the min/max value - if (pInfo->stage == 0) { - if (pCtx->isAggSet) { - double tmin = 0.0, tmax = 0.0; - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_INT64_VAL(&pCtx->agg.min); - tmax = (double)GET_INT64_VAL(&pCtx->agg.max); - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - tmin = GET_DOUBLE_VAL(&pCtx->agg.min); - tmax = GET_DOUBLE_VAL(&pCtx->agg.max); - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_UINT64_VAL(&pCtx->agg.min); - tmax = (double)GET_UINT64_VAL(&pCtx->agg.max); - } else { - assert(true); - } - - if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) { - SET_DOUBLE_VAL(&pInfo->minval, tmin); - } - - if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) { - SET_DOUBLE_VAL(&pInfo->maxval, tmax); - } - - pInfo->numOfElems += (pCtx->size - pCtx->agg.numOfNull); - } else { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, data); - - if (v < GET_DOUBLE_VAL(&pInfo->minval)) { - SET_DOUBLE_VAL(&pInfo->minval, v); - } - - if (v > GET_DOUBLE_VAL(&pInfo->maxval)) { - SET_DOUBLE_VAL(&pInfo->maxval, v); - } - - pInfo->numOfElems += 1; - } - } - - return; - } - - // the second stage, calculate the true percentile value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - tMemBucketPut(pInfo->pMemBucket, data, 1); - } - - SET_VAL(pCtx, notNullElems, 1); - //pResInfo->hasResult = DATA_SET_FLAG; -} - -static void percentile_finalizer(SqlFunctionCtx *pCtx) { -// double v = pCtx->param[0].param.nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].param.i : pCtx->param[0].param.d; - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); - - tMemBucket * pMemBucket = ppInfo->pMemBucket; - if (pMemBucket == NULL || pMemBucket->total == 0) { // check for null - assert(ppInfo->numOfElems == 0); - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else { -// SET_DOUBLE_VAL((double *)pCtx->pOutput, getPercentile(pMemBucket, v)); - } - - tMemBucketDestroy(pMemBucket); - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////// -static void buildHistogramInfo(SAPercentileInfo* pInfo) { - pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo)); - pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); -} - -static SAPercentileInfo *getAPerctInfo(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo* pInfo = NULL; - - if (pCtx->stableQuery && pCtx->scanFlag != MERGE_STAGE) { - pInfo = (SAPercentileInfo*) pCtx->pOutput; - } else { - pInfo = GET_ROWCELL_INTERBUF(pResInfo); - } - - buildHistogramInfo(pInfo); - return pInfo; -} - -static bool apercentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - - char *tmp = (char *)pInfo + sizeof(SAPercentileInfo); - pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN); - return true; -} - -static void apercentile_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - - assert(pInfo->pHisto->elems != NULL); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, data); - tHistogramAdd(&pInfo->pHisto, v); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void apercentile_func_merge(SqlFunctionCtx *pCtx) { - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); - - pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); - pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - - if (pInput->pHisto->numOfElems <= 0) { - return; - } - - SAPercentileInfo *pOutput = getAPerctInfo(pCtx); - SHistogramInfo *pHisto = pOutput->pHisto; - - if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - } else { - //TODO(dengyihao): avoid memcpy - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - tHistogramDestroy(&pRes); - } - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - SET_VAL(pCtx, 1, 1); -} - -static void apercentile_finalizer(SqlFunctionCtx *pCtx) { - double v = (pCtx->param[0].param.nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].param.i : pCtx->param[0].param.d; - - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->scanFlag == MERGE_STAGE) { -// if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null -// assert(pOutput->pHisto->numOfElems > 0); -// -// double ratio[] = {v}; -// double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); -// -// memcpy(pCtx->pOutput, res, sizeof(double)); -// taosMemoryFree(res); -// } else { -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// return; -// } - } else { - if (pOutput->pHisto->numOfElems > 0) { - double ratio[] = {v}; - - double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - memcpy(pCtx->pOutput, res, sizeof(double)); - taosMemoryFree(res); - } else { // no need to free - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - } - - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////// -static bool leastsquares_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // 2*3 matrix -// pInfo->startVal = pCtx->param[0].param.d; - return true; -} - -#define LEASTSQR_CAL(p, x, y, index, step) \ - do { \ - (p)[0][0] += (double)(x) * (x); \ - (p)[0][1] += (double)(x); \ - (p)[0][2] += (double)(x) * (y)[index]; \ - (p)[1][2] += (y)[index]; \ - (x) += step; \ - } while (0) - -#define LEASTSQR_CAL_LOOP(ctx, param, x, y, tsdbType, n, step) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&(y)[i], tsdbType)) { \ - continue; \ - } \ - (n)++; \ - LEASTSQR_CAL(param, x, y, i, step); \ - } - -static void leastsquares_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - double(*param)[3] = pInfo->mat; - double x = pInfo->startVal; - - void *pData = GET_INPUT_DATA_LIST(pCtx); - - int32_t numOfElem = 0; - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *p = pData; - // LEASTSQR_CAL_LOOP(pCtx, param, pParamData, p); - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) p, pCtx->inputType)) { - continue; - } - - param[0][0] += x * x; - param[0][1] += x; - param[0][2] += x * p[i]; - param[1][2] += p[i]; - - x += pCtx->param[1].param.d; - numOfElem++; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - }; - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - int8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - } - - pInfo->startVal = x; - pInfo->num += numOfElem; - - if (pInfo->num > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } - - SET_VAL(pCtx, numOfElem, 1); -} - -static void leastsquares_finalizer(SqlFunctionCtx *pCtx) { - // no data in query - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pInfo->num == 0) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - double(*param)[3] = pInfo->mat; - - param[1][1] = (double)pInfo->num; - param[1][0] = param[0][1]; - - param[0][0] -= param[1][0] * (param[0][1] / param[1][1]); - param[0][2] -= param[1][2] * (param[0][1] / param[1][1]); - param[0][1] = 0; - param[1][2] -= param[0][2] * (param[1][0] / param[0][0]); - param[1][0] = 0; - param[0][2] /= param[0][0]; - - param[1][2] /= param[1][1]; - - int32_t maxOutputSize = AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE; - size_t n = snprintf(varDataVal(pCtx->pOutput), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}", - param[0][2], param[1][2]); - - varDataSetLen(pCtx->pOutput, n); - doFinalizer(pCtx); -} - -static void date_col_output_function(SqlFunctionCtx *pCtx) { - SET_VAL(pCtx, pCtx->size, 1); - *(int64_t *)(pCtx->pOutput) = pCtx->startTs; -} - -static void col_project_function(SqlFunctionCtx *pCtx) { - // the number of output rows should not affect the final number of rows, so set it to be 0 - if (pCtx->numOfParams == 2) { - return; - } - - // only one row is required. -// if (pCtx->param[0].param.i == 1) { -// SET_VAL(pCtx, pCtx->size, 1); -// } else { -// INC_INIT_VAL(pCtx, pCtx->size); -// } - - char *pData = GET_INPUT_DATA_LIST(pCtx); - if (pCtx->order == TSDB_ORDER_ASC) { -// int32_t numOfRows = (pCtx->param[0].param.i == 1)? 1:pCtx->size; -// memcpy(pCtx->pOutput, pData, (size_t) numOfRows * pCtx->inputBytes); - } else { - for(int32_t i = 0; i < pCtx->size; ++i) { - memcpy(pCtx->pOutput + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, - pCtx->inputBytes); - } - } -} - -/** - * only used for tag projection query in select clause - * @param pCtx - * @return - */ -static void tag_project_function(SqlFunctionCtx *pCtx) { - INC_INIT_VAL(pCtx, pCtx->size); - - assert(pCtx->inputBytes == pCtx->resDataInfo.bytes); - - taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->resDataInfo.type, true); - char* data = pCtx->pOutput; - pCtx->pOutput += pCtx->resDataInfo.bytes; - - // directly copy from the first one - for (int32_t i = 1; i < pCtx->size; ++i) { - memmove(pCtx->pOutput, data, pCtx->resDataInfo.bytes); - pCtx->pOutput += pCtx->resDataInfo.bytes; - } -} - -/** - * used in group by clause. when applying group by tags, the tags value is - * assign by using tag function. - * NOTE: there is only ONE output for ONE query range - * @param pCtx - * @return - */ -static void copy_function(SqlFunctionCtx *pCtx); - -static void tag_function(SqlFunctionCtx *pCtx) { - SET_VAL(pCtx, 1, 1); - if (pCtx->scanFlag == MERGE_STAGE) { - copy_function(pCtx); - } else { - taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->resDataInfo.type, true); - } -} - -static void copy_function(SqlFunctionCtx *pCtx) { - SET_VAL(pCtx, pCtx->size, 1); - - char *pData = GET_INPUT_DATA_LIST(pCtx); - assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); -} - -enum { - INITIAL_VALUE_NOT_ASSIGNED = 0, -}; - -static bool diff_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - // diff function require the value is set to -1 - pCtx->param[1].param.nType = INITIAL_VALUE_NOT_ASSIGNED; - return false; -} - -static bool deriv_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - // diff function require the value is set to -1 - SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResultInfo); - -// pDerivInfo->ignoreNegative = pCtx->param[1].param.i; - pDerivInfo->prevTs = -1; -// pDerivInfo->tsWindow = pCtx->param[0].param.i; - pDerivInfo->valueSet = false; - return false; -} - -static void deriv_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo); - - void *data = GET_INPUT_DATA_LIST(pCtx); - - int32_t notNullElems = 0; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - - TSKEY *pTimestamp = NULL;//pCtx->pTsOutput; - TSKEY *tsList = GET_TS_LIST(pCtx); - - double *pOutput = (double *)pCtx->pOutput; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *pData = (int32_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - SET_DOUBLE_VAL(pOutput, ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs)); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - - break; - }; - - case TSDB_DATA_TYPE_BIGINT: { - int64_t *pData = (int64_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = (double) pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *pData = (double *)data; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_FLOAT: { - float *pData = (float *)data; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *pData = (int16_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t *pData = (int8_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - default: - assert(0); -// qError("error input type"); - } - - GET_RES_INFO(pCtx)->numOfRes += notNullElems; -} - -// TODO difference in date column -static void diff_function(SqlFunctionCtx *pCtx) { - void *data = GET_INPUT_DATA_LIST(pCtx); - bool isFirstBlock = (pCtx->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED); - - int32_t notNullElems = 0; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - - TSKEY* pTimestamp = NULL;//pCtx->pTsOutput; - TSKEY* tsList = GET_TS_LIST(pCtx); - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *pData = (int32_t *)data; - int32_t *pOutput = (int32_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int32_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - }; - case TSDB_DATA_TYPE_BIGINT: { - int64_t *pData = (int64_t *)data; - int64_t *pOutput = (int64_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = pData[i] - pCtx->param[1].param.i; // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *pData = (double *)data; - double *pOutput = (double *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].param.d); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.d = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *pData = (float *)data; - float *pOutput = (float *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (float)(pData[i] - pCtx->param[1].param.d); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.d = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *pData = (int16_t *)data; - int16_t *pOutput = (int16_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int16_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t *pData = (int8_t *)data; - int8_t *pOutput = (int8_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int8_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - default: - assert(0); -// qError("error input type"); - } - - // initial value is not set yet - if (pCtx->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { - /* - * 1. current block and blocks before are full of null - * 2. current block may be null value - */ - assert(pCtx->hasNull); - } else { - int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems; - - GET_RES_INFO(pCtx)->numOfRes += forwardStep; - } -} - -#if 0 -char *getArithColumnData(void *param, const char* name, int32_t colId) { - SScalarFunctionSupport *pSupport = (SScalarFunctionSupport *)param; - - int32_t index = -1; - for (int32_t i = 0; i < pSupport->numOfCols; ++i) { - if (colId == pSupport->colList[i].colId) { - index = i; - break; - } - } - - assert(index >= 0); - return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; -} -#endif - -static void arithmetic_function(SqlFunctionCtx *pCtx) { - GET_RES_INFO(pCtx)->numOfRes += pCtx->size; - //SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; - -// SScalarParam output = {0}; -// output.data = pCtx->pOutput; - - //evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); -} - -#define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ - { \ - type *inputData = (type *)data; \ - for (int32_t i = 0; i < elemCnt; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&inputData[i], tsdbType)) { \ - continue; \ - } \ - if (inputData[i] < minOutput) { \ - minOutput = (double)inputData[i]; \ - } \ - if (inputData[i] > maxOutput) { \ - maxOutput = (double)inputData[i]; \ - } \ - numOfNotNullElem++; \ - } \ - } - -///////////////////////////////////////////////////////////////////////////////// -static bool spread_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // this is the server-side setup function in client-side, the secondary merge do not need this procedure - if (pCtx->scanFlag == MERGE_STAGE) { -// pCtx->param[0].param.d = DBL_MAX; -// pCtx->param[3].param.d = -DBL_MAX; - } else { - pInfo->min = DBL_MAX; - pInfo->max = -DBL_MAX; - } - - return true; -} - -static void spread_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - int32_t numOfElems = 0; - - // todo : opt with pre-calculated result - // column missing cause the hasNull to be true - if (pCtx->isAggSet) { - numOfElems = pCtx->size - pCtx->agg.numOfNull; - - // all data are null in current data block, ignore current data block - if (numOfElems == 0) { - goto _spread_over; - } - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType) || IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType) || - (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { - if (pInfo->min > pCtx->agg.min) { - pInfo->min = (double)pCtx->agg.min; - } - - if (pInfo->max < pCtx->agg.max) { - pInfo->max = (double)pCtx->agg.max; - } - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - if (pInfo->min > GET_DOUBLE_VAL((const char *)&(pCtx->agg.min))) { - pInfo->min = GET_DOUBLE_VAL((const char *)&(pCtx->agg.min)); - } - - if (pInfo->max < GET_DOUBLE_VAL((const char *)&(pCtx->agg.max))) { - pInfo->max = GET_DOUBLE_VAL((const char *)&(pCtx->agg.max)); - } - } - - goto _spread_over; - } - - void *pData = GET_INPUT_DATA_LIST(pCtx); - numOfElems = 0; - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int8_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int16_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int32_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT || pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int64_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, double, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, float, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint8_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint16_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint32_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint64_t, pCtx->inputType, numOfElems); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == numOfElems); - } - - _spread_over: - SET_VAL(pCtx, numOfElems, 1); - - if (numOfElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - pInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); - } -} - -/* - * here we set the result value back to the intermediate buffer, to apply the finalize the function - * the final result is generated in spread_function_finalizer - */ -void spread_func_merge(SqlFunctionCtx *pCtx) { - SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); - if (pData->hasResult != DATA_SET_FLAG) { - return; - } - -// if (pCtx->param[0].param.d > pData->min) { -// pCtx->param[0].param.d = pData->min; -// } - -// if (pCtx->param[3].param.d < pData->max) { -// pCtx->param[3].param.d = pData->max; -// } - -// GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -void spread_function_finalizer(SqlFunctionCtx *pCtx) { - /* - * here we do not check the input data types, because in case of metric query, - * the type of intermediate data is binary - */ - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->scanFlag == MERGE_STAGE) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - -// if (pResInfo->hasResult != DATA_SET_FLAG) { -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// return; -// } - -// SET_DOUBLE_VAL((double *)pCtx->pOutput, pCtx->param[3].param.d - pCtx->param[0].param.d); - } else { - assert(IS_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); - - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->max - pInfo->min); - } - - GET_RES_INFO(pCtx)->numOfRes = 1; // todo add test case - doFinalizer(pCtx); -} - - -/** - * param[1]: start time - * param[2]: end time - * @param pCtx - */ -static bool twa_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->p.key = INT64_MIN; - pInfo->win = TSWINDOW_INITIALIZER; - return true; -} - -static double twa_get_area(SPoint1 s, SPoint1 e) { - if ((s.val >= 0 && e.val >= 0)|| (s.val <=0 && e.val <= 0)) { - return (s.val + e.val) * (e.key - s.key) / 2; - } - - double x = (s.key * e.val - e.key * s.val)/(e.val - s.val); - double val = (s.val * (x - s.key) + e.val * (e.key - x)) / 2; - return val; -} - -static int32_t twa_function_impl(SqlFunctionCtx* pCtx, int32_t index, int32_t size) { - int32_t notNullElems = 0; - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *tsList = GET_TS_LIST(pCtx); - - int32_t i = index; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - SPoint1* last = &pInfo->p; - - if (pCtx->start.key != INT64_MIN) { - assert((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || - (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); - - assert(last->key == INT64_MIN); - - last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); - - pInfo->dOutput += twa_get_area(pCtx->start, *last); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pCtx->start.key; - notNullElems++; - i += step; - } else if (pInfo->p.key == INT64_MIN) { - last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = last->key; - notNullElems++; - i += step; - } - - // calculate the value of - switch(pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t *val = (uint8_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t *val = (uint16_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t *val = (uint32_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t *val = (uint64_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double) val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - default: assert(0); - } - - // the last interpolated time window value - if (pCtx->end.key != INT64_MIN) { - pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end); - pInfo->p = pCtx->end; - } - - pInfo->win.ekey = pInfo->p.key; - return notNullElems; -} - -static void twa_function(SqlFunctionCtx *pCtx) { - void *data = GET_INPUT_DATA_LIST(pCtx); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // skip null value - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); - while (pCtx->hasNull && i < pCtx->size && i >= 0 && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { - i += step; - } - - int32_t notNullElems = 0; - if (i >= 0 && i < pCtx->size) { - notNullElems = twa_function_impl(pCtx, i, pCtx->size); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } - - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, pInfo, sizeof(STwaInfo)); - } -} - -/* - * To copy the input to interResBuf to avoid the input buffer space be over writen - * by next input data. The TWA function only applies to each table, so no merge procedure - * is required, we simply copy to the resut ot interResBuffer. - */ -void twa_function_copy(SqlFunctionCtx *pCtx) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); -// pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; -} - -void twa_function_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); - if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - return; - } - -// assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult); - if (pInfo->win.ekey == pInfo->win.skey) { - SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->p.val); - } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput , pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey)); - } - - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -/** - * - * @param pCtx - */ - -static void interp_function_impl(SqlFunctionCtx *pCtx) { - int32_t type = (int32_t) pCtx->param[2].param.i; - if (type == TSDB_FILL_NONE) { - return; - } - - bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); - - if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - *(TSKEY *)pCtx->pOutput = pCtx->startTs; - } else if (type == TSDB_FILL_NULL) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else if (type == TSDB_FILL_SET_VALUE) { -// taosVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); - } else { - if (pCtx->start.key != INT64_MIN && ((ascQuery && pCtx->start.key <= pCtx->startTs && pCtx->end.key >= pCtx->startTs) || ((!ascQuery) && pCtx->start.key >= pCtx->startTs && pCtx->end.key <= pCtx->startTs))) { - if (type == TSDB_FILL_PREV) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val); - } else { - assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->resDataInfo.bytes, pCtx->inputType); - } - } else if (type == TSDB_FILL_NEXT) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->end.val); - } else { - assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->resDataInfo.bytes, pCtx->inputType); - } - } else if (type == TSDB_FILL_LINEAR) { - SPoint point1 = {.key = pCtx->start.key, .val = &pCtx->start.val}; - SPoint point2 = {.key = pCtx->end.key, .val = &pCtx->end.val}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; - - int32_t srcType = pCtx->inputType; - if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? - if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } else { -// taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); - } - } else { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } - } - } else { - // no data generated yet - if (pCtx->size < 1) { - return; - } - - // check the timestamp in input buffer - TSKEY skey = GET_TS_DATA(pCtx, 0); - - if (type == TSDB_FILL_PREV) { - if ((ascQuery && skey > pCtx->startTs) || ((!ascQuery) && skey < pCtx->startTs)) { - return; - } - - if (pCtx->size > 1) { - TSKEY ekey = GET_TS_DATA(pCtx, 1); - if ((ascQuery && ekey > skey && ekey <= pCtx->startTs) || - ((!ascQuery) && ekey < skey && ekey >= pCtx->startTs)){ - skey = ekey; - } - } -// assignVal(pCtx->pOutput, pCtx->pInput, pCtx->resDataInfo.bytes, pCtx->inputType); - } else if (type == TSDB_FILL_NEXT) { - TSKEY ekey = skey; - char* val = NULL; - - if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) { - if (pCtx->size > 1) { - ekey = GET_TS_DATA(pCtx, 1); - if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) { - return; - } - - val = ((char*)pCtx->pInput) + pCtx->inputBytes; - } else { - return; - } - } else { - val = (char*)pCtx->pInput; - } - - assignVal(pCtx->pOutput, val, pCtx->resDataInfo.bytes, pCtx->inputType); - } else if (type == TSDB_FILL_LINEAR) { - if (pCtx->size <= 1) { - return; - } - - TSKEY ekey = GET_TS_DATA(pCtx, 1); - - // no data generated yet - if ((ascQuery && !(skey <= pCtx->startTs && ekey >= pCtx->startTs)) - || ((!ascQuery) && !(skey >= pCtx->startTs && ekey <= pCtx->startTs))) { - return; - } - - char *start = GET_INPUT_DATA(pCtx, 0); - char *end = GET_INPUT_DATA(pCtx, 1); - - SPoint point1 = {.key = skey, .val = start}; - SPoint point2 = {.key = ekey, .val = end}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; - - int32_t srcType = pCtx->inputType; - if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? - if (isNull(start, srcType) || isNull(end, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } else { -// taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, srcType); - } - } else { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } - } - } - } - - SET_VAL(pCtx, 1, 1); -} - -static void interp_function(SqlFunctionCtx *pCtx) { - // at this point, the value is existed, return directly - if (pCtx->size > 0) { - bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); - TSKEY key; - char *pData; - int32_t typedData = 0; - - if (ascQuery) { - key = GET_TS_DATA(pCtx, 0); - pData = GET_INPUT_DATA(pCtx, 0); - } else { - key = pCtx->start.key; - if (key == INT64_MIN) { - key = GET_TS_DATA(pCtx, 0); - pData = GET_INPUT_DATA(pCtx, 0); - } else { - if (!(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL)) { - pData = pCtx->start.ptr; - } else { - typedData = 1; - pData = (char *)&pCtx->start.val; - } - } - } - - //if (key == pCtx->startTs && (ascQuery || !(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL))) { - if (key == pCtx->startTs) { - if (typedData) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, *(double *)pData); - } else { - assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); - } - - SET_VAL(pCtx, 1, 1); - } else { - interp_function_impl(pCtx); - } - } else { //no qualified data rows and interpolation is required - interp_function_impl(pCtx); - } -} - -static bool ts_comp_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; // not initialized since it has been initialized - } - - STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->pTSBuf = tsBufCreate(false, pCtx->order); - pInfo->pTSBuf->tsOrder = pCtx->order; - return true; -} - -static void ts_comp_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; - - const char *input = GET_INPUT_DATA_LIST(pCtx); - - // primary ts must be existed, so no need to check its existance - if (pCtx->order == TSDB_ORDER_ASC) { -// tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].param.i, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); - } else { - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *d = GET_INPUT_DATA(pCtx, i); -// tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].param.i, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); - } - } - - SET_VAL(pCtx, pCtx->size, 1); - //pResInfo->hasResult = DATA_SET_FLAG; -} - -static void ts_comp_finalize(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - STSBuf * pTSbuf = pInfo->pTSBuf; - - tsBufFlush(pTSbuf); -// qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); - - // TODO refactor transfer ownership of current file - *(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile; - - pResInfo->complete = true; - - // get the file size - int64_t file_size; - if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) { - pResInfo->numOfRes = (uint32_t )file_size; - } - - pTSbuf->remainOpen = true; - tsBufDestroy(pTSbuf); - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////////////// -// rate functions -static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) { - if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey) || - (pRateInfo->firstKey >= pRateInfo->lastKey)) { - return 0.0; - } - - double diff = 0; - if (pRateInfo->isIRate) { - // If the previous value of the last is greater than the last value, only keep the last point instead of the delta - // value between two values. - diff = pRateInfo->lastValue; - if (diff >= pRateInfo->firstValue) { - diff -= pRateInfo->firstValue; - } - } else { - diff = pRateInfo->correctionValue + pRateInfo->lastValue - pRateInfo->firstValue; - if (diff <= 0) { - return 0; - } - } - - int64_t duration = pRateInfo->lastKey - pRateInfo->firstKey; - if (duration == 0) { - return 0; - } - - return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; -} - -static bool rate_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SRateInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->correctionValue = 0; - pInfo->firstKey = INT64_MIN; - pInfo->lastKey = INT64_MIN; - pInfo->firstValue = (double) INT64_MIN; - pInfo->lastValue = (double) INT64_MIN; - - pInfo->hasResult = 0; - pInfo->isIRate = (pCtx->functionId == FUNCTION_IRATE); - return true; -} - -static void rate_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - -// qDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *pData = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { -// qDebug("%p rate_function() index of null data:%d", pCtx, i); - continue; - } - - notNullElems++; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; - } - - if (INT64_MIN == pRateInfo->lastValue) { - pRateInfo->lastValue = v; - } else if (v < pRateInfo->lastValue) { - pRateInfo->correctionValue += pRateInfo->lastValue; - } - - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[i]; - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pRateInfo->hasResult = DATA_SET_FLAG; -// pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - -static void rate_func_copy(SqlFunctionCtx *pCtx) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); -// pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; -} - -static void rate_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pRateInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - return; - } - -// SET_DOUBLE_VAL((double*) pCtx->pOutput, do_calc_rate(pRateInfo, (double) TSDB_TICK_PER_SECOND(pCtx->param[0].param.i))); - - // cannot set the numOfIteratedElems again since it is set during previous iteration - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; - - doFinalizer(pCtx); -} - -static void irate_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *pData = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - continue; - } - - notNullElems++; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if ((INT64_MIN == pRateInfo->lastKey) || primaryKey[i] > pRateInfo->lastKey) { - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[i]; - continue; - } - - if ((INT64_MIN == pRateInfo->firstKey) || primaryKey[i] > pRateInfo->firstKey) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; - break; - } - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pRateInfo->hasResult = DATA_SET_FLAG; -// pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - -static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDistInfo* pDist) { - SBufferReader br = tbufInitReader(data, len, false); - - pDist->numOfTables = tbufReadUint32(&br); - pDist->numOfFiles = tbufReadUint16(&br); - pDist->totalSize = tbufReadUint64(&br); - pDist->totalRows = tbufReadUint64(&br); - pDist->maxRows = tbufReadInt32(&br); - pDist->minRows = tbufReadInt32(&br); - pDist->numOfInmemRows = tbufReadUint32(&br); - pDist->numOfSmallBlocks = tbufReadUint32(&br); - int64_t numSteps = tbufReadUint64(&br); - - bool comp = tbufReadUint8(&br); - uint32_t compLen = tbufReadUint32(&br); - - size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo)); - - char* outputBuf = NULL; - if (comp) { - outputBuf = taosMemoryMalloc(originalLen); - - size_t actualLen = compLen; - const char* compStr = tbufReadBinary(&br, &actualLen); - - int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, - (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); - assert(orignalLen == numSteps *sizeof(SFileBlockInfo)); - } else { - outputBuf = (char*) tbufReadBinary(&br, &originalLen); - } - - pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); - if (comp) { - taosMemoryFreeClear(outputBuf); - } -} - -static void blockInfo_func(SqlFunctionCtx* pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); - - int32_t len = *(int32_t*) pCtx->pInput; - blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); -// pDist->rowSize = (uint16_t)pCtx->param[0].param.i; - - memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); - - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; -} - -static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDistInfo* pSrc) { - STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); - assert(pDist != NULL && pSrc != NULL); - - pDist->numOfTables += pSrc->numOfTables; - pDist->numOfInmemRows += pSrc->numOfInmemRows; - pDist->numOfSmallBlocks += pSrc->numOfSmallBlocks; - pDist->numOfFiles += pSrc->numOfFiles; - pDist->totalSize += pSrc->totalSize; - pDist->totalRows += pSrc->totalRows; - -// if (pResInfo->hasResult == DATA_SET_FLAG) { -// pDist->maxRows = TMAX(pDist->maxRows, pSrc->maxRows); -// pDist->minRows = TMIN(pDist->minRows, pSrc->minRows); -// } else { - pDist->maxRows = pSrc->maxRows; - pDist->minRows = pSrc->minRows; - - int32_t maxSteps = TSDB_MAX_MAXROWS_FBLOCK/TSDB_BLOCK_DIST_STEP_ROWS; - if (TSDB_MAX_MAXROWS_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { - ++maxSteps; - } - pDist->dataBlockInfos = taosArrayInit(maxSteps, sizeof(SFileBlockInfo)); - taosArraySetSize(pDist->dataBlockInfos, maxSteps); -// } - - size_t steps = taosArrayGetSize(pSrc->dataBlockInfos); - for (int32_t i = 0; i < steps; ++i) { - int32_t srcNumBlocks = ((SFileBlockInfo*)taosArrayGet(pSrc->dataBlockInfos, i))->numBlocksOfStep; - SFileBlockInfo* blockInfo = (SFileBlockInfo*)taosArrayGet(pDist->dataBlockInfos, i); - blockInfo->numBlocksOfStep += srcNumBlocks; - } -} - -void block_func_merge(SqlFunctionCtx* pCtx) { - STableBlockDistInfo info = {0}; - int32_t len = *(int32_t*) pCtx->pInput; - blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - mergeTableBlockDist(pResInfo, &info); - taosArrayDestroy(info.dataBlockInfos); - - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; -} - -void getPercentiles(STableBlockDistInfo *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, - double* percents, int32_t* percentiles) { - if (totalBlocks == 0) { - for (int32_t i = 0; i < numOfPercents; ++i) { - percentiles[i] = 0; - } - return; - } - - SArray *blocksInfos = pTableBlockDist->dataBlockInfos; - size_t numSteps = taosArrayGetSize(blocksInfos); - size_t cumulativeBlocks = 0; - - int percentIndex = 0; - for (int32_t indexStep = 0; indexStep < numSteps; ++indexStep) { - int32_t numStepBlocks = ((SFileBlockInfo *)taosArrayGet(blocksInfos, indexStep))->numBlocksOfStep; - if (numStepBlocks == 0) continue; - cumulativeBlocks += numStepBlocks; - - while (percentIndex < numOfPercents) { - double blockRank = totalBlocks * percents[percentIndex]; - if (blockRank <= cumulativeBlocks) { - percentiles[percentIndex] = indexStep; - ++percentIndex; - } else { - break; - } - } - } - - for (int32_t i = 0; i < numOfPercents; ++i) { - percentiles[i] = (percentiles[i]+1) * TSDB_BLOCK_DIST_STEP_ROWS - TSDB_BLOCK_DIST_STEP_ROWS/2; - } -} - -void generateBlockDistResult(STableBlockDistInfo *pTableBlockDist, char* result) { - if (pTableBlockDist == NULL) { - return; - } - - SArray* blockInfos = pTableBlockDist->dataBlockInfos; - uint64_t totalRows = pTableBlockDist->totalRows; - size_t numSteps = taosArrayGetSize(blockInfos); - int64_t totalBlocks = 0; - int64_t min = -1, max = -1, avg = 0; - - for (int32_t i = 0; i < numSteps; i++) { - SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t blocks = blockInfo->numBlocksOfStep; - totalBlocks += blocks; - } - - avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; - min = totalBlocks > 0 ? pTableBlockDist->minRows : 0; - max = totalBlocks > 0 ? pTableBlockDist->maxRows : 0; - - double stdDev = 0; - if (totalBlocks > 0) { - double variance = 0; - for (int32_t i = 0; i < numSteps; i++) { - SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t blocks = blockInfo->numBlocksOfStep; - int32_t rows = (i * TSDB_BLOCK_DIST_STEP_ROWS + TSDB_BLOCK_DIST_STEP_ROWS / 2); - variance += blocks * (rows - avg) * (rows - avg); - } - variance = variance / totalBlocks; - stdDev = sqrt(variance); - } - - double percents[] = {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99}; - int32_t percentiles[] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; - assert(sizeof(percents)/sizeof(double) == sizeof(percentiles)/sizeof(int32_t)); - getPercentiles(pTableBlockDist, totalBlocks, sizeof(percents)/sizeof(double), percents, percentiles); - - uint64_t totalLen = pTableBlockDist->totalSize; - int32_t rowSize = pTableBlockDist->rowSize; - int32_t smallBlocks = pTableBlockDist->numOfSmallBlocks; - double compRatio = (totalRows>0) ? ((double)(totalLen)/(rowSize*totalRows)) : 1; - int sz = sprintf(result + VARSTR_HEADER_SIZE, - "summary: \n\t " - "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t " - "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t " - "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t " - "Rows=[%"PRIu64"], Blocks=[%"PRId64"], SmallBlocks=[%d], Size=[%.3f(Kb)] Comp=[%.2f]\n\t " - "RowsInMem=[%d] \n\t", - percentiles[0], percentiles[1], percentiles[2], percentiles[3], percentiles[4], percentiles[5], - percentiles[6], percentiles[7], percentiles[8], percentiles[9], percentiles[10], percentiles[11], - min, max, avg, stdDev, - totalRows, totalBlocks, smallBlocks, totalLen/1024.0, compRatio, - pTableBlockDist->numOfInmemRows); - varDataSetLen(result, sz); - UNUSED(sz); -} - -void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); - -// pDist->rowSize = (uint16_t)pCtx->param[0].param.i; - generateBlockDistResult(pDist, pCtx->pOutput); - - if (pDist->dataBlockInfos != NULL) { - taosArrayDestroy(pDist->dataBlockInfos); - pDist->dataBlockInfos = NULL; - } - - // cannot set the numOfIteratedElems again since it is set during previous iteration - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; - - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////////////////// -/* - * function compatible list. - * tag and ts are not involved in the compatibility check - * - * 1. functions that are not simultaneously present with any other functions. e.g., diff/ts_z/top/bottom - * 2. functions that are only allowed to be present only with same functions. e.g., last_row, interp - * 3. functions that are allowed to be present with other functions. - * e.g., count/sum/avg/min/max/stddev/percentile/apercentile/first/last... - * - */ -int32_t functionCompatList[] = { - // count, sum, avg, min, max, stddev, percentile, apercentile, first, last - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // last_row,top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_comp - 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, - // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, stddev_dst, interp rate irate - 1, 1, 1, 1, -1, 1, 1, 1, 5, 1, 1, - // tid_tag, derivative, blk_info - 6, 8, 7, -}; -#endif diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c deleted file mode 100644 index 039e0a9dfc..0000000000 --- a/source/libs/function/src/texpr.c +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "function.h" -#include "os.h" - -#include "texception.h" -#include "tmsg.h" - -static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)); - -void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { - if (pNode == NULL) { - return; - } - - if (pNode->nodeType == TEXPR_BINARYEXPR_NODE || pNode->nodeType == TEXPR_UNARYEXPR_NODE) { - doExprTreeDestroy(&pNode, fp); - } - taosMemoryFree(pNode); -} - -static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { - if (*pExpr == NULL) { - return; - } - taosMemoryFree(*pExpr); - *pExpr = NULL; -} - -// TODO: these three functions should be made global -static void* exception_calloc(size_t nmemb, size_t size) { - void* p = taosMemoryCalloc(nmemb, size); - if (p == NULL) { - THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - return p; -} - -static void* exception_malloc(size_t size) { - void* p = taosMemoryMalloc(size); - if (p == NULL) { - THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - return p; -} - -static UNUSED_FUNC char* exception_strdup(const char* str) { - char* p = strdup(str); - if (p == NULL) { - THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - return p; -} - diff --git a/source/libs/function/src/tfunctionInt.c b/source/libs/function/src/tfunctionInt.c new file mode 100644 index 0000000000..85719e428b --- /dev/null +++ b/source/libs/function/src/tfunctionInt.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taosdef.h" +#include "tmsg.h" +#include "thash.h" +#include "ttypes.h" + +#include "function.h" +#include "tbuffer.h" +#include "tcompression.h" +#include "tdatablock.h" +#include "tfunctionInt.h" +#include "thistogram.h" +#include "tpercentile.h" +#include "ttszip.h" +#include "tudf.h" + +void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { + pCell->initialized = false; +} + +int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock) { + int32_t maxRows = 0; + + for (int32_t j = 0; j < num; ++j) { +#if 0 + int32_t id = pCtx[j].functionId; + + /* + * ts, tag, tagprj function can not decide the output number of current query + * the number of output result is decided by main output + */ + if (id == FUNCTION_TS || id == FUNCTION_TAG || id == FUNCTION_TAGPRJ) { + continue; + } +#endif + SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); + if (pResInfo != NULL && maxRows < pResInfo->numOfRes) { + maxRows = pResInfo->numOfRes; + } + } + + assert(maxRows >= 0); + + blockDataEnsureCapacity(pResBlock, maxRows); + for(int32_t i = 0; i < num; ++i) { + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); + + SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[i]); + if (pResInfo->numOfRes == 0) { + for(int32_t j = 0; j < pResInfo->numOfRes; ++j) { + colDataAppend(pCol, j, NULL, true); // TODO add set null data api + } + } else { + for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { + colDataAppend(pCol, j, GET_ROWCELL_INTERBUF(pResInfo), false); + } + } + } + + pResBlock->info.rows = maxRows; + return maxRows; +} + +bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { + assert(pEntry != NULL); + return pEntry->complete; +} + +bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry) { + return pEntry->initialized; +} From a87692372a0142253229e04de918785f2f35c74d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Aug 2022 20:16:04 +0800 Subject: [PATCH 138/165] refactor: do some internal refactor. --- source/libs/function/src/tfunctionInt.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/source/libs/function/src/tfunctionInt.c b/source/libs/function/src/tfunctionInt.c index 85719e428b..ff15b7714f 100644 --- a/source/libs/function/src/tfunctionInt.c +++ b/source/libs/function/src/tfunctionInt.c @@ -37,17 +37,6 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock int32_t maxRows = 0; for (int32_t j = 0; j < num; ++j) { -#if 0 - int32_t id = pCtx[j].functionId; - - /* - * ts, tag, tagprj function can not decide the output number of current query - * the number of output result is decided by main output - */ - if (id == FUNCTION_TS || id == FUNCTION_TAG || id == FUNCTION_TAGPRJ) { - continue; - } -#endif SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); if (pResInfo != NULL && maxRows < pResInfo->numOfRes) { maxRows = pResInfo->numOfRes; From cfb61d665d9b743fd9dcf78ef2d9b17d0002b2f0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 10 Aug 2022 20:44:48 +0800 Subject: [PATCH 139/165] fix: row iter --- include/common/trow.h | 2 +- source/common/src/trow.c | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index a1db2ef708..ad9f52390e 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -256,7 +256,7 @@ static FORCE_INLINE void *tdGetBitmapAddrTp(STSRow *pRow, uint32_t flen) { static FORCE_INLINE void *tdGetBitmapAddrKv(STSRow *pRow, col_id_t nKvCols) { // The primary TS key is stored separatedly and is Norm value, thus should minus 1 firstly - return POINTER_SHIFT(TD_ROW_COL_IDX(pRow), nKvCols * sizeof(SKvRowIdx)); + return POINTER_SHIFT(TD_ROW_COL_IDX(pRow), (--nKvCols) * sizeof(SKvRowIdx)); } void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols); int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index b13e5f1ca8..d81fbf4629 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -427,7 +427,7 @@ bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVa tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); return true; } - int16_t nCols = tdRowGetNCols(pRow); + int16_t nCols = tdRowGetNCols(pRow) - 1; if (nCols <= 0) { pVal->valType = TD_VTYPE_NONE; return true; @@ -488,7 +488,7 @@ bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCe STSRow *pRow = pIter->pRow; SKvRowIdx *pKvIdx = NULL; bool colFound = false; - col_id_t kvNCols = tdRowGetNCols(pRow); + col_id_t kvNCols = tdRowGetNCols(pRow) - 1; void *pColIdx = TD_ROW_COL_IDX(pRow); while (*nIdx < kvNCols) { @@ -534,7 +534,7 @@ bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, return TSDB_CODE_SUCCESS; } - if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; return terrno; } @@ -814,7 +814,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP - TASSERT(colIdx < tdRowGetNCols(pRow)); + TASSERT(colIdx < tdRowGetNCols(pRow) - 1); if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { output->valType = TD_VTYPE_NONE; return terrno; @@ -1054,14 +1054,14 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { break; case TD_ROW_KV: #ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols - 1); + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps); #endif len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + pBuilder->nBoundBitmaps; // add TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); - TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols - 1); + TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); break; default: TASSERT(0); From 1420524dd0f9737575f6f12f027fef15a3b5c43d Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 10 Aug 2022 20:57:37 +0800 Subject: [PATCH 140/165] Update 01-docker.md --- docs/zh/05-get-started/01-docker.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index 741f6dfeab..9a8552826d 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -2,6 +2,9 @@ sidebar_label: Docker title: 通过 Docker 快速体验 TDengine --- +:::info +如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. +::: 本节首先介绍如何通过 Docker 快速体验 TDengine，然后介绍如何在 Docker 环境下体验 TDengine 的写入和查询功能。 From 7f4b46e0031d749a0eeb1d226332c138f7c68b88 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 10 Aug 2022 21:17:38 +0800 Subject: [PATCH 141/165] feat(stream): session window trigger delete --- include/common/tmsg.h | 20 ++++++++- include/common/tmsgdef.h | 1 + source/common/src/tmsg.c | 39 ++++++++++++++++- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/mnode/impl/src/mndStream.c | 17 +++++--- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/sma/smaTimeRange.c | 7 +-- source/dnode/vnode/src/tq/tqSink.c | 47 +++++++++++++++++---- source/dnode/vnode/src/vnd/vnodeSvr.c | 21 +++++++++ 9 files changed, 134 insertions(+), 21 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d26e99f423..9fe01b0cf6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1368,8 +1368,8 @@ typedef struct { int32_t numOfCols; int64_t skey; int64_t ekey; - int64_t version; // for stream - TSKEY watermark;// for stream + int64_t version; // for stream + TSKEY watermark; // for stream char data[]; } SRetrieveTableRsp; @@ -3079,6 +3079,22 @@ typedef struct SDeleteRes { int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); int32_t tDecodeDeleteRes(SDecoder* pCoder, SDeleteRes* pRes); +typedef struct { + int64_t uid; + int64_t ts; +} SSingleDeleteReq; + +int32_t tEncodeSSingleDeleteReq(SEncoder* pCoder, const SSingleDeleteReq* pReq); +int32_t tDecodeSSingleDeleteReq(SDecoder* pCoder, SSingleDeleteReq* pReq); + +typedef struct { + int64_t suid; + SArray* deleteReqs; // SArray +} SBatchDeleteReq; + +int32_t tEncodeSBatchDeleteReq(SEncoder* pCoder, const SBatchDeleteReq* pReq); +int32_t tDecodeSBatchDeleteReq(SDecoder* pCoder, SBatchDeleteReq* pReq); + typedef struct { int32_t msgIdx; int32_t msgType; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 9ddb872007..6462c7afbf 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -202,6 +202,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) TD_DEF_MSG_TYPE(TDMT_VND_FETCH_RSMA, "vnode-fetch-rsma", SRSmaFetchMsg, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) + TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index b630419fc4..7dd3ce34c3 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -305,7 +305,7 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) taosArrayPush(desc.subDesc, &sDesc); } } - + ASSERT(desc.subPlanNum == taosArrayGetSize(desc.subDesc)); taosArrayPush(pReq->query->queryDesc, &desc); @@ -5770,3 +5770,40 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { } return 0; } + +int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) { + if (tEncodeI64(pEncoder, pReq->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->ts) < 0) return -1; + return 0; +} + +int32_t tDecodeSSingleDeleteReq(SDecoder *pDecoder, SSingleDeleteReq *pReq) { + if (tDecodeI64(pDecoder, &pReq->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->ts) < 0) return -1; + return 0; +} + +int32_t tEncodeSBatchDeleteReq(SEncoder *pEncoder, const SBatchDeleteReq *pReq) { + if (tEncodeI64(pEncoder, pReq->suid) < 0) return -1; + int32_t sz = taosArrayGetSize(pReq->deleteReqs); + if (tEncodeI32(pEncoder, sz) < 0) return -1; + for (int32_t i = 0; i < sz; i++) { + SSingleDeleteReq *pOneReq = taosArrayGet(pReq->deleteReqs, i); + if (tEncodeSSingleDeleteReq(pEncoder, pOneReq) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) { + if (tDecodeI64(pDecoder, &pReq->suid) < 0) return -1; + int32_t sz; + if (tDecodeI32(pDecoder, &sz) < 0) return -1; + pReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + if (pReq->deleteReqs == NULL) return -1; + for (int32_t i = 0; i < sz; i++) { + SSingleDeleteReq deleteReq; + if (tDecodeSSingleDeleteReq(pDecoder, &deleteReq) < 0) return -1; + taosArrayPush(pReq->deleteReqs, &deleteReq); + } + return 0; +} diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 1b799b1e5e..d13daffa08 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -364,6 +364,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_CHECK_ALTER_INFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b172b00fab..b4af39e467 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -943,12 +943,17 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false); - char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n)); - varDataSetLen(targetSTB, strlen(varDataVal(targetSTB))); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); + if (pStream->targetSTbName[0] == 0) { + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, NULL, true); + } else { + char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n)); + varDataSetLen(targetSTB, strlen(varDataVal(targetSTB))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); + } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7b22e061f0..ffc1966733 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -172,7 +172,7 @@ int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list); SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId); + const char* stbFullName, int32_t vgId, SBatchDeleteReq* pDeleteReq); // sma int32_t smaInit(); diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 7f69acc889..8da397f0c3 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -119,7 +119,7 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { SName stbFullName = {0}; tNameFromString(&stbFullName, pCfg->dstTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); SVCreateStbReq pReq = {0}; - pReq.name = (char*)tNameGetTableName(&stbFullName); + pReq.name = (char *)tNameGetTableName(&stbFullName); pReq.suid = pCfg->dstTbUid; pReq.schemaRow = pCfg->schemaRow; pReq.schemaTag = pCfg->schemaTag; @@ -200,8 +200,9 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { goto _err; } + SBatchDeleteReq deleteReq; SSubmitReq *pSubmitReq = tdBlockToSubmit((const SArray *)msg, pTsmaStat->pTSchema, true, pTsmaStat->pTSma->dstTbUid, - pTsmaStat->pTSma->dstTbName, pTsmaStat->pTSma->dstVgId); + pTsmaStat->pTSma->dstTbName, pTsmaStat->pTSma->dstVgId, &deleteReq); if (!pSubmitReq) { smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), @@ -230,4 +231,4 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { _err: tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 6b4d5a2f3e..2b3283a395 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -16,7 +16,7 @@ #include "tq.h" SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId) { + const char* stbFullName, int32_t vgId, SBatchDeleteReq* deleteReq) { SSubmitReq* ret = NULL; SArray* schemaReqs = NULL; SArray* schemaReqSz = NULL; @@ -33,10 +33,13 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo schemaReqSz = taosArrayInit(sz, sizeof(int32_t)); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - STagVal tagVal = { - .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .i64 = (int64_t)pDataBlock->info.groupId, + if (pDataBlock->info.type == STREAM_DELETE_DATA) { + // + } + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, }; STag* pTag = NULL; taosArrayClear(tagArray); @@ -176,17 +179,45 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo } void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { - const SArray* pRes = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; + const SArray* pRes = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + SBatchDeleteReq deleteReq = {0}; tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, (int32_t)pRes->size); ASSERT(pTask->tbSink.pTSchema); + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); SSubmitReq* pReq = tdBlockToSubmit(pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid, - pTask->tbSink.stbFullName, pVnode->config.vgId); + pTask->tbSink.stbFullName, pVnode->config.vgId, &deleteReq); tqDebug("vgId:%d, task %d convert blocks over, put into write-queue", TD_VID(pVnode), pTask->taskId); + int32_t code; + int32_t len; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* buf = taosMemoryCalloc(1, len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + + if (taosArrayGetSize(deleteReq.deleteReqs) != 0) { + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = buf, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + taosArrayDestroy(deleteReq.deleteReqs); + /*tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema);*/ // build write msg SRpcMsg msg = { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e4e1e608e5..1f13bde0c1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -29,6 +29,7 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; @@ -190,6 +191,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp case TDMT_VND_DELETE: if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; + case TDMT_VND_BATCH_DEL: + if (vnodeProcessBatchDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + break; /* TQ */ case TDMT_VND_MQ_VG_CHANGE: if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), @@ -1053,6 +1057,23 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void return 0; } +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SBatchDeleteReq deleteReq; + SDecoder decoder; + tDecoderInit(&decoder, pReq, len); + tDecodeSBatchDeleteReq(&decoder, &deleteReq); + + int32_t sz = taosArrayGetSize(deleteReq.deleteReqs); + for (int32_t i = 0; i < sz; i++) { + SSingleDeleteReq *pOneReq = taosArrayGet(deleteReq.deleteReqs, i); + int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, pOneReq->uid, pOneReq->ts, pOneReq->ts); + if (code) { + // TODO + } + } + return 0; +} + static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; From e03adb140af0ebf9c90510d911c79d1e8ca5bd1a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 10 Aug 2022 21:42:30 +0800 Subject: [PATCH 142/165] feat: remove temporary interface declarations (#15940) * add a few stmt_ interface declartion * add taos_load_table_info * add taos_set_config(() interface * add more interfaces * warn 'reset query cache' only since lack engine support * update taos-tools * feat: remove temporary interface declaration * fix: remove temporary interfaces --- include/client/taos.h | 13 ------ source/client/src/TSDBJNIConnector.c | 61 +--------------------------- source/client/src/clientMain.c | 15 ------- 3 files changed, 1 insertion(+), 88 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index b7df0e4d29..dd7266bd96 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -196,15 +196,6 @@ DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, vo DLL_EXPORT void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); DLL_EXPORT const void *taos_get_raw_block(TAOS_RES *res); -// Shuduo: temporary enable for app build -#if 1 -typedef void (*__taos_sub_fn_t)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); -DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, __taos_sub_fn_t fp, - void *param, int interval); -DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub); -DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); -#endif - DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); @@ -281,10 +272,6 @@ DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res); /* ------------------------------ TMQ END -------------------------------- */ -#if 1 // Shuduo: temporary enable for app build -typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); -#endif - typedef enum { TSDB_SRV_STATUS_UNAVAILABLE = 0, TSDB_SRV_STATUS_NETWORK_OK = 1, diff --git a/source/client/src/TSDBJNIConnector.c b/source/client/src/TSDBJNIConnector.c index 38bbec24ce..37041da695 100644 --- a/source/client/src/TSDBJNIConnector.c +++ b/source/client/src/TSDBJNIConnector.c @@ -611,65 +611,6 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeConnectionIm } } -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNIEnv *env, jobject jobj, jlong con, - jboolean restart, jstring jtopic, - jstring jsql, jint jinterval) { - jlong sub = 0; - TAOS *taos = (TAOS *)con; - char *topic = NULL; - char *sql = NULL; - - jniGetGlobalMethod(env); - jniDebug("jobj:%p, in TSDBJNIConnector_subscribeImp", jobj); - - if (jtopic != NULL) { - topic = (char *)(*env)->GetStringUTFChars(env, jtopic, NULL); - } - if (jsql != NULL) { - sql = (char *)(*env)->GetStringUTFChars(env, jsql, NULL); - } - - if (topic == NULL || sql == NULL) { - jniDebug("jobj:%p, invalid argument: topic or sql is NULL", jobj); - return sub; - } - - TAOS_SUB *tsub = taos_subscribe(taos, (int)restart, topic, sql, NULL, NULL, jinterval); - sub = (jlong)tsub; - - if (sub == 0) { - jniDebug("jobj:%p, failed to subscribe: topic:%s", jobj, topic); - } else { - jniDebug("jobj:%p, successfully subscribe: topic: %s", jobj, topic); - } - - (*env)->ReleaseStringUTFChars(env, jtopic, topic); - (*env)->ReleaseStringUTFChars(env, jsql, sql); - - return sub; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub) { - jniDebug("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%lld", jobj, sub); - jniGetGlobalMethod(env); - - TAOS_SUB *tsub = (TAOS_SUB *)sub; - TAOS_RES *res = taos_consume(tsub); - - if (res == NULL) { - jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub); - return 0l; - } - - return (jlong)res; -} - -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, - jboolean keepProgress) { - TAOS_SUB *tsub = (TAOS_SUB *)sub; - taos_unsubscribe(tsub, keepProgress); -} - JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTableSqlImp(JNIEnv *env, jobject jobj, jlong con, jbyteArray jsql) { TAOS *tscon = (TAOS *)con; @@ -1087,4 +1028,4 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsert return JNI_OUT_OF_MEMORY; } return (jlong)tres; -} \ No newline at end of file +} diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 0d41d35721..0ec724c6d0 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -939,21 +939,6 @@ const void *taos_get_raw_block(TAOS_RES *res) { return pRequest->body.resInfo.pData; } -TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, - void *param, int interval) { - // TODO - return NULL; -} - -TAOS_RES *taos_consume(TAOS_SUB *tsub) { - // TODO - return NULL; -} - -void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { - // TODO -} - int taos_load_table_info(TAOS *taos, const char *tableNameList) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; From 0aba651a08cc151167151e689383a5e7ce24e5f0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 10 Aug 2022 22:08:21 +0800 Subject: [PATCH 143/165] chore: update taosa and rust connector for3.0 (#15958) * chore: add libtaos-ws for 3.0 * chore: update taosws-rs * chore: add libtaosws to install/remove script * chore: update taosws-rs * chore: update taosws-rs * chore: update taos-tools, taosws-rs for 3.0 * fix: packaging/tools/make_install.sh for 3.0 * chore: update taos-tools * chore: fix release script for 3.0 * chore: update taosws-rs for 3.0 * chore: add taows-rs submodule for 3.0 * chore: update taosws-rs for 3.0 * fix: install script support taosws for 3.0 * fix: script error handle for 3.0 * chore: update taosws-rs for 3.0 fix segfault * chore: change container_build for websocket build * fix: install script for taosws * fix: . * chore: update taosws-rs for 3.0 * chore: update taosws-rs for 3.0 * chore: update tools/CMakeLists.txt to allow compile taosws-rw on any platform * chore: taosws 648cc62 for 3.0 * chore: update taosws 29424d5 for 3.0 * chore: update cmake/taosws_CMakeLists.txt.in with new repo/commit * chore: update taosa 3d21433 and rust connector for new raw block --- cmake/taosadapter_CMakeLists.txt.in | 2 +- cmake/taosws_CMakeLists.txt.in | 2 +- tools/CMakeLists.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index ee1e7d3b31..ed8216be91 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG ed6a160 + GIT_TAG 3d21433 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index c6d42b686c..506559a245 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 97c4bac + GIT_TAG 7a54d21 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1f066daabe..5751c347e3 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -112,7 +112,7 @@ ELSE () COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND - COMMAND wget -c https://github.com/upx/upx/releases/download/v3.96/upx-3.96-${PLATFORM_ARCH_STR}_linux.tar.xz -O $ENV{HOME}/upx.tar.xz && tar -xvJf $ENV{HOME}/upx.tar.xz -C $ENV{HOME}/ --strip-components 1 > /dev/null && $ENV{HOME}/upx taosadapter || : + COMMAND wget -nc https://github.com/upx/upx/releases/download/v3.96/upx-3.96-${PLATFORM_ARCH_STR}_linux.tar.xz -O $ENV{HOME}/upx.tar.xz && tar -xvJf $ENV{HOME}/upx.tar.xz -C $ENV{HOME}/ --strip-components 1 > /dev/null && $ENV{HOME}/upx taosadapter || : COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ From 8675580758c3ae36f7cc25b123ddac8903892d38 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 10 Aug 2022 23:00:38 +0800 Subject: [PATCH 144/165] docs: update connector for 3.0 (#15957) --- .../14-reference/03-connector/03-connector.mdx | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/zh/14-reference/03-connector/03-connector.mdx b/docs/zh/14-reference/03-connector/03-connector.mdx index 7a4a85276e..e18759e600 100644 --- a/docs/zh/14-reference/03-connector/03-connector.mdx +++ b/docs/zh/14-reference/03-connector/03-connector.mdx @@ -10,14 +10,14 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 目前 TDengine 的原生接口连接器可支持的平台包括：X64/X86/ARM64/ARM32/MIPS/Alpha 等硬件平台，以及 Linux/Win64/Win32 等开发环境。对照矩阵如下： -| **CPU** | **OS** | **JDBC** | **Python** | **Go** | **Node.js** | **C#** | **Rust** | C/C++ | +| **CPU** | **OS** | **Java** | **Python** | **Go** | **Node.js** | **C#** | **Rust** | C/C++ | | -------------- | --------- | -------- | ---------- | ------ | ----------- | ------ | -------- | ----- | | **X86 64bit** | **Linux** | ● | ● | ● | ● | ● | ● | ● | | **X86 64bit** | **Win64** | ● | ● | ● | ● | ● | ● | ● | | **X86 64bit** | **Win32** | ● | ● | ● | ● | ○ | ○ | ● | | **X86 32bit** | **Win32** | ○ | ○ | ○ | ○ | ○ | ○ | ● | | **ARM64** | **Linux** | ● | ● | ● | ● | ○ | ○ | ● | -| **ARM32** | **Linux** | ● | ● | ● | ● | ○ | ○ | ● | +| **ARM32** | **Linux** | ○ | ○ | ○ | ○ | ○ | ○ | ● | | **MIPS 龙芯** | **Linux** | ○ | ○ | ○ | ○ | ○ | ○ | ○ | | **Alpha 申威** | **Linux** | ○ | ○ | -- | -- | -- | -- | ○ | | **X86 海光** | **Linux** | ○ | ○ | ○ | -- | -- | -- | ○ | @@ -32,6 +32,7 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | **TDengine 版本** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | | --------------------- | -------- | ---------- | ------------ | ------------- | --------------- | -------- | +| **3.0.0.0 及以上** | 3.0.0 | 当前版本 | 3.0 分支 | 3.0.0 | 3.0.0 | 当前版本 | | **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | | **2.4.0.6 及以上** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | | **2.4.0.4 - 2.4.0.5** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | @@ -50,7 +51,7 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **连续查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **订阅功能** | 支持 | 支持 | 支持 | 支持 | 支持 | 暂不支持 | +| ** TMQ ** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **Schemaless** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | @@ -58,17 +59,17 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 由于不同编程语言数据库框架规范不同，并不意味着所有 C/C++ 接口都需要对应封装支持。 ::: -### 使用 REST 接口 +### 使用 http (REST 或 WebSocket) 接口 | **功能特性** | **Java** | **Python** | **Go** | **C#（暂不支持）** | **Node.js** | **Rust** | | ------------------------------ | -------- | ---------- | -------- | ------------------ | ----------- | -------- | | **连接管理** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | | **普通查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | | **连续查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | -| **参数绑定** | 不支持 | 不支持 | 不支持 | N/A | 不支持 | 不支持 | -| **订阅功能** | 不支持 | 不支持 | 不支持 | N/A | 不支持 | 不支持 | -| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | N/A | 暂不支持 | 暂不支持 | -| **批量拉取（基于 WebSocket）** | 支持 | 暂不支持 | 暂不支持 | N/A | 暂不支持 | 暂不支持 | +| **参数绑定** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 | +| ** TMQ ** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 | +| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 暂不支持 | +| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 暂不支持 | N/A | 不支持 | 支持 | | **DataFrame** | 不支持 | 支持 | 不支持 | N/A | 不支持 | 不支持 | :::warning From a5ef854e87e3f1051baeb5f415f3f904acd79976 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 11 Aug 2022 07:45:49 +0800 Subject: [PATCH 145/165] row: code optimization --- include/common/trow.h | 23 ++-- source/common/src/trow.c | 186 ++++++++++++++++++++--------- source/dnode/vnode/src/tq/tqRead.c | 2 +- 3 files changed, 138 insertions(+), 73 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index ad9f52390e..6cc9355144 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -195,12 +195,6 @@ static FORCE_INLINE void tdRowSetVal(SCellVal *pVal, uint8_t valType, void *val) pVal->valType = valType; pVal->val = val; } -/** - * @brief Primary TS column not included. - * - * @param pRow - * @return FORCE_INLINE - */ static FORCE_INLINE col_id_t tdRowGetNCols(STSRow *pRow) { return *(col_id_t *)TD_ROW_NCOLS(pRow); } static FORCE_INLINE void tdRowCpy(void *dst, const STSRow *pRow) { memcpy(dst, pRow, TD_ROW_LEN(pRow)); } static FORCE_INLINE const char *tdRowEnd(STSRow *pRow) { return (const char *)POINTER_SHIFT(pRow, TD_ROW_LEN(pRow)); } @@ -325,20 +319,17 @@ typedef struct { col_id_t kvIdx; // [0, nKvCols) } STSRowIter; -void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); -void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema); +void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema); +void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); +bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); +bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal); + int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow); -bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); -bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal); -bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal); -bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, - SCellVal *pVal); -bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal); -void tdSCellValPrint(SCellVal *pVal, int8_t colType); +bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag); #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_ROW_H_*/ +#endif /*_TD_COMMON_ROW_H_*/ \ No newline at end of file diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d81fbf4629..24032c8df2 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -32,9 +32,13 @@ const uint8_t tdVTypeByte[2][3] = {{ }; // declaration -static uint8_t tdGetBitmapByte(uint8_t byte); -static int32_t tdCompareColId(const void *arg1, const void *arg2); -static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2); +static uint8_t tdGetBitmapByte(uint8_t byte); +static bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); +static bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal); +static bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, + col_id_t colIdx, SCellVal *pVal); +static bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal); +static void tdSCellValPrint(SCellVal *pVal, int8_t colType); // implementation /** @@ -330,7 +334,6 @@ void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterReset(&iter, row); printf("%s >>>type:%d,sver:%d ", tag, (int32_t)TD_ROW_TYPE(row), (int32_t)TD_ROW_SVER(row)); - STColumn *cols = (STColumn *)&iter.pSchema->columns; while (true) { SCellVal sVal = {.valType = 255, NULL}; @@ -340,7 +343,6 @@ void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); tdSCellValPrint(&sVal, cols[iter.colIdx - 1].type); } - printf("\n"); } @@ -422,6 +424,16 @@ void tdSCellValPrint(SCellVal *pVal, int8_t colType) { } } +static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { + if (*(col_id_t *)key1 > ((SKvRowIdx *)key2)->colId) { + return 1; + } else if (*(col_id_t *)key1 < ((SKvRowIdx *)key2)->colId) { + return -1; + } else { + return 0; + } +} + bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); @@ -458,6 +470,39 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } +bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + return true; + } + + if (TD_IS_TP_ROW(pIter->pRow)) { + STColumn *pCol = NULL; + STSchema *pSchema = pIter->pSchema; + while (pIter->colIdx < pSchema->numOfCols) { + pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key + if (colId == pCol->colId) { + break; + } else if (pCol->colId < colId) { + ++pIter->colIdx; + continue; + } else { + return false; + } + } + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + ++pIter->colIdx; + } else if (TD_IS_KV_ROW(pIter->pRow)) { + return tdSTSRowIterGetKvVal(pIter, colId, &pIter->kvIdx, pVal); + } else { + pVal->valType = TD_VTYPE_NONE; + terrno = TSDB_CODE_INVALID_PARA; + if (COL_REACH_END(colId, pIter->maxColId)) return false; + } + return true; +} + bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { if (pIter->colIdx >= pIter->pSchema->numOfCols) { return false; @@ -473,9 +518,10 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { } if (TD_IS_TP_ROW(pIter->pRow)) { - tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); } else if (TD_IS_KV_ROW(pIter->pRow)) { - tdGetKvRowValOfColEx(pIter, pCol->colId, &pIter->kvIdx, pVal); + tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal); + ASSERT(0); } else { ASSERT(0); } @@ -484,45 +530,7 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { return true; } -bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal) { - STSRow *pRow = pIter->pRow; - SKvRowIdx *pKvIdx = NULL; - bool colFound = false; - col_id_t kvNCols = tdRowGetNCols(pRow) - 1; - void *pColIdx = TD_ROW_COL_IDX(pRow); - - while (*nIdx < kvNCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, *nIdx * sizeof(SKvRowIdx)); - if (pKvIdx->colId == colId) { - ++(*nIdx); - pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); - colFound = true; - break; - } else if (pKvIdx->colId > colId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - ++(*nIdx); - } - } - - if (!colFound) { - if (colId <= pIter->maxColId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - return false; - } - } - - if (tdGetBitmapValType(pIter->pBitmap, (*nIdx) - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { - pVal->valType = TD_VTYPE_NONE; - } - - return true; -} - -bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { +bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { STSRow *pRow = pIter->pRow; if (pRow->statis == 0) { pVal->valType = TD_VTYPE_NORM; @@ -550,14 +558,41 @@ bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, return true; } -static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { - if (*(col_id_t *)key1 > ((SKvRowIdx *)key2)->colId) { - return 1; - } else if (*(col_id_t *)key1 < ((SKvRowIdx *)key2)->colId) { - return -1; - } else { - return 0; +bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal) { + STSRow *pRow = pIter->pRow; + SKvRowIdx *pKvIdx = NULL; + bool colFound = false; + col_id_t kvNCols = tdRowGetNCols(pRow) - 1; + void *pColIdx = TD_ROW_COL_IDX(pRow); + while (*nIdx < kvNCols) { + pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, *nIdx * sizeof(SKvRowIdx)); + if (pKvIdx->colId == colId) { + ++(*nIdx); + pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); + colFound = true; + break; + } else if (pKvIdx->colId > colId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + ++(*nIdx); + } } + + if (!colFound) { + if (colId <= pIter->maxColId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + return false; + } + } + + if (tdGetBitmapValType(pIter->pBitmap, pIter->kvIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + pVal->valType = TD_VTYPE_NONE; + } + + return true; } int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { @@ -673,7 +708,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { return 0; } -static int32_t tdCompareColId(const void *arg1, const void *arg2) { +static FORCE_INLINE int32_t tdCompareColId(const void *arg1, const void *arg2) { int32_t colId = *(int32_t *)arg1; STColumn *pCol = (STColumn *)arg2; @@ -686,6 +721,45 @@ static int32_t tdCompareColId(const void *arg1, const void *arg2) { } } +bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + return true; + } + + STSRow *pRow = pIter->pRow; + int16_t colIdx = -1; + if (TD_IS_TP_ROW(pRow)) { + STSchema *pSchema = pIter->pSchema; + STColumn *pCol = + (STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ); + if (!pCol) { + pVal->valType = TD_VTYPE_NONE; + if (COL_REACH_END(colId, pIter->maxColId)) return false; + return true; + } +#ifdef TD_SUPPORT_BITMAP + colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); +#endif + tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); + } else if (TD_IS_KV_ROW(pRow)) { + SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), + compareKvRowColId, TD_EQ); +#ifdef TD_SUPPORT_BITMAP + if (pIdx) { + colIdx = POINTER_DISTANCE(pIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); + } +#endif + tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); + } else { + if (COL_REACH_END(colId, pIter->maxColId)) return false; + pVal->valType = TD_VTYPE_NONE; + } + + return true; +} + int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { TASSERT(0); @@ -1250,7 +1324,7 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { pIter->pRow = pRow; pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); pIter->offset = 0; - pIter->colIdx = 0; + pIter->colIdx = 0; // PRIMARYKEY_TIMESTAMP_COL_ID; pIter->kvIdx = 0; } @@ -1288,4 +1362,4 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value); } -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index ced98c8afa..5d7814a045 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -325,7 +325,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { for (int32_t i = 0; i < colActual; i++) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SCellVal sVal = {0}; - if (!tdSTSRowIterNext(&iter, &sVal)) { + if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; } if (colDataAppend(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { From 9891f78961b30c53162f915206b753e6c2cf5c6d Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 11 Aug 2022 09:43:03 +0800 Subject: [PATCH 146/165] fix: can not clear result info when in stream computing --- source/libs/function/src/builtinsimpl.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 0fd543953d..d44cb952e3 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -498,8 +498,7 @@ int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; - cleanupResultRowEntry(pResInfo); + pResInfo->isNullRes = (pResInfo->isNullRes == 1) ? 1 : (pResInfo->numOfRes == 0);; char* in = finalResult; colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); From c1539bc064613368dd8c9875cc9eb26f5964e05e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 11 Aug 2022 09:45:47 +0800 Subject: [PATCH 147/165] enh: optimize getting table hash performance --- source/libs/catalog/src/ctgUtil.c | 42 ++++++++++++++++++------- source/libs/executor/src/executorimpl.c | 2 ++ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 1ca60c89cd..5b3ae013bd 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -883,6 +883,20 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } +int32_t ctgHashValueComp(void const *lp, void const *rp) { + uint32_t *key = (uint32_t *)lp; + SVgroupInfo *pVg = *(SVgroupInfo **)rp; + + if (*key < pVg->hashBegin) { + return -1; + } else if (*key > pVg->hashEnd) { + return 1; + } + + return 0; +} + + int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; @@ -923,9 +937,17 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo } } + taosHashCancelIterate(dbInfo->vgHash, pIter); return TSDB_CODE_SUCCESS; } + SArray* pVgList = taosArrayInit(vgNum, POINTER_BYTES); + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + taosArrayPush(pVgList, &pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); + } + char tbFullName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFullName, "%s.", dbFName); int32_t offset = strlen(tbFullName); @@ -940,25 +962,19 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo uint32_t hashValue = (*fp)(tbFullName, (uint32_t)tbNameLen); - void *pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - vgInfo = pIter; - if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { - taosHashCancelIterate(dbInfo->vgHash, pIter); - break; - } - - pIter = taosHashIterate(dbInfo->vgHash, pIter); - vgInfo = NULL; - } + SVgroupInfo **p = taosArraySearch(pVgList, &hashValue, ctgHashValueComp, TD_EQ); - if (NULL == vgInfo) { + if (NULL == p) { ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, taosHashGetSize(dbInfo->vgHash)); + taosArrayDestroy(pVgList); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } + vgInfo = *p; + SVgroupInfo* pNewVg = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == pNewVg) { + taosArrayDestroy(pVgList); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -977,6 +993,8 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo } } + taosArrayDestroy(pVgList); + CTG_RET(code); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7e77799702..16b7a9efb7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1962,6 +1962,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); if (pExchangeInfo == NULL) { qWarn("failed to acquire exchange operator, since it may have been released"); + taosMemoryFree(pMsg->pData); return TSDB_CODE_SUCCESS; } @@ -1980,6 +1981,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { ASSERT(pRsp != NULL); qDebug("%s fetch rsp received, index:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfRows); } else { + taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); } From 446b3291e76b91a5463cbb3d71b58ad8e35f6a9a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 11 Aug 2022 09:53:51 +0800 Subject: [PATCH 148/165] fix: remove obsolete code --- source/common/src/trow.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 24032c8df2..565498a47b 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -521,7 +521,6 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); } else if (TD_IS_KV_ROW(pIter->pRow)) { tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal); - ASSERT(0); } else { ASSERT(0); } From 44123ec8f0bf0946fafde605d436b18c3c415d60 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 11 Aug 2022 10:19:06 +0800 Subject: [PATCH 149/165] feat(stream):optimzie stream interval --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executorimpl.c | 6 + source/libs/executor/src/scanoperator.c | 9 +- source/libs/executor/src/timewindowoperator.c | 103 ++++++++++++------ source/libs/stream/src/streamUpdate.c | 2 +- 5 files changed, 83 insertions(+), 38 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d460644633..d5711087e7 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -1017,6 +1017,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); +bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6de77f63f4..3ac08bc20e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1277,8 +1277,12 @@ void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo) { } void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset) { + bool init = false; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultEntryInfo(pResult, i, rowEntryInfoOffset); + if (init) { + continue; + } struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { @@ -1295,6 +1299,8 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } else { pResInfo->initialized = true; } + } else { + init = true; } } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2f6d04f684..d8de8df163 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1174,10 +1174,15 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[rowId], &pInfo->interval, TSDB_ORDER_ASC); + bool isClosed = false; + STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; + if (isOverdue(tsCol[rowId], &pInfo->twAggSup)) { + win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[rowId], &pInfo->interval, TSDB_ORDER_ASC); + isClosed = isCloseWindow(&win, &pInfo->twAggSup); + } // must check update info first. bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); - if ((update || (isSignleIntervalWindow(pInfo) && isCloseWindow(&win, &pInfo->twAggSup) && + if ((update || (isSignleIntervalWindow(pInfo) && isClosed && isDeletedWindow(&win, pBlock->info.groupId, pInfo->sessionSup.pIntervalAggSup))) && out) { appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 610303dc50..548bc046c2 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -851,23 +851,34 @@ static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t g return TSDB_CODE_SUCCESS; } +static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SHashObj* pUpdatedMap) { + SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + if (newPos == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + newPos->groupId = groupId; + newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset}; + *(int64_t*)newPos->key = ts; + SWinRes key = {.ts = ts, .groupId = groupId}; + if (taosHashPut(pUpdatedMap, &key, sizeof(SWinRes), &newPos, sizeof(void*)) != TSDB_CODE_SUCCESS) { + taosMemoryFree(newPos); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj* pUpdatedMap) { + return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap);; +} + static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) { return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated); } -static void removeResult(SArray* pUpdated, SWinRes* pKey) { - int32_t size = taosArrayGetSize(pUpdated); - int32_t index = binarySearchCom(pUpdated, size, pKey, TSDB_ORDER_DESC, compareResKey); - if (index >= 0 && 0 == compareResKey(pKey, pUpdated, index)) { - taosArrayRemove(pUpdated, index); - } -} - -static void removeResults(SArray* pWins, SArray* pUpdated) { +static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) { int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { SWinRes* pW = taosArrayGet(pWins, i); - removeResult(pUpdated, pW); + taosHashRemove(pUpdatedMap, pW, sizeof(SWinRes)); } } @@ -894,11 +905,14 @@ int32_t compareWinRes(void* pKey, void* data, int32_t index) { return -1; } -static void removeDeleteResults(SArray* pUpdated, SArray* pDelWins) { - int32_t upSize = taosArrayGetSize(pUpdated); +static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { + if (!pUpdatedMap || taosHashGetSize(pUpdatedMap) == 0) { + return; + } int32_t delSize = taosArrayGetSize(pDelWins); - for (int32_t i = 0; i < upSize; i++) { - SResKeyPos* pResKey = taosArrayGetP(pUpdated, i); + void* pIte = NULL; + while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { + SResKeyPos* pResKey = (SResKeyPos*)pIte; int32_t index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes); if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) { taosArrayRemove(pDelWins, index); @@ -914,7 +928,7 @@ bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) { bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); } static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, - int32_t scanFlag, SArray* pUpdated) { + int32_t scanFlag, SHashObj* pUpdatedMap) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -940,7 +954,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResultRow(pResult, tableGroupId, pUpdated); + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); } } @@ -997,7 +1011,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResultRow(pResult, tableGroupId, pUpdated); + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); } @@ -1437,7 +1451,7 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* } } -static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) { +static int32_t getAllIntervalWindow(SHashObj* pHashMap, SHashObj* resWins) { void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { @@ -1446,7 +1460,7 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) { ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); SResultRowPosition* pPos = (SResultRowPosition*)pIte; - int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, resWins); + int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1455,7 +1469,7 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) { } static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, - SHashObj* pPullDataMap, SArray* closeWins, SArray* pRecyPages, + SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pRecyPages, SDiskbasedBuf* pDiscBuf) { qDebug("===stream===close interval window"); void* pIte = NULL; @@ -1487,7 +1501,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, } SResultRowPosition* pPos = (SResultRowPosition*)pIte; if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins); + int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, closeWins); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1577,11 +1591,14 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* downstream = pOperator->pDownstream[0]; SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP); + SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { break; } + // qInfo("===stream===%ld", pBlock->info.version); printDataBlock(pBlock, "single interval recv"); if (pBlock->info.type == STREAM_CLEAR) { @@ -1594,7 +1611,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); continue; } @@ -1617,17 +1634,24 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); } pOperator->status = OP_RES_TO_RETURN; - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdated, + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + void* pIte = NULL; + while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { + taosArrayPush(pUpdated, pIte); + } + taosHashCleanup(pUpdatedMap); + taosArraySort(pUpdated, resultrowComparAsc); + finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - removeDeleteResults(pUpdated, pInfo->pDelWins); + removeDeleteResults(pUpdatedMap, pInfo->pDelWins); doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows > 0) { return pInfo->pDelRes; @@ -2831,7 +2855,7 @@ STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { } static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, - SArray* pUpdated) { + SHashObj* pUpdatedMap) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -2913,8 +2937,8 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); } - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) { - saveResultRow(pResult, tableGroupId, pUpdated); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); @@ -3020,6 +3044,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; SOperatorInfo* downstream = pOperator->pDownstream[0]; SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP); + SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); TSKEY maxTs = INT64_MIN; SExprSupp* pSup = &pOperator->exprSupp; @@ -3077,7 +3103,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { clearSpecialDataBlock(pInfo->pUpdateRes); - removeDeleteResults(pUpdated, pInfo->pDelWins); + removeDeleteResults(pUpdatedMap, pInfo->pDelWins); pOperator->status = OP_RES_TO_RETURN; qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); break; @@ -3104,7 +3130,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { taosArrayDestroy(pUpWins); continue; } - removeResults(pUpWins, pUpdated); + removeResults(pUpWins, pUpdatedMap); copyDataBlock(pInfo->pUpdateRes, pBlock); // copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex); pInfo->returnUpdate = true; @@ -3122,15 +3148,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated); continue; } - removeResults(pInfo->pDelWins, pUpdated); + removeResults(pInfo->pDelWins, pUpdatedMap); break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes)); doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); - removeResults(pUpWins, pUpdated); + removeResults(pUpWins, pUpdatedMap); taosArrayDestroy(pUpWins); if (taosArrayGetSize(pUpdated) > 0) { break; @@ -3146,7 +3172,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true); - doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated); + doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -3171,12 +3197,19 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); if (IS_FINAL_OP(pInfo)) { closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap, - pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + pUpdatedMap, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); } else { pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; } + void* pIte = NULL; + while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { + taosArrayPush(pUpdated, pIte); + } + taosHashCleanup(pUpdatedMap); + taosArraySort(pUpdated, resultrowComparAsc); + finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index c686fa05ce..0b1ce27b77 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -124,7 +124,7 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; pInfo->pCloseWinSBF = NULL; - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK); pInfo->maxVersion = 0; pInfo->scanGroupId = 0; From 76ef53e1eeebd75d05ba256e522b035d152da696 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 11 Aug 2022 11:11:08 +0800 Subject: [PATCH 150/165] feat: update taostools 57bdfbf for3.0 (#15959) * feat: update taos-tools for 3.0 [TD-14141] * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools 8e3b3ee * fix: remove submodules * feat: update taos-tools c529299 * feat: update taos-tools 9dc2fec for 3.0 * fix: optim upx * feat: update taos-tools f4e456a for 3.0 * feat: update taos-tools 2a2def1 for 3.0 * feat: update taos-tools c9cc20f for 3.0 * feat: update taostoosl 8a5e336 for 3.0 * feat: update taostools 3c7dafe for 3.0 * feat: update taos-tools 2d68404 for 3.0 * feat: update taos-tools 57bdfbf for 3.0 * fix: jenkinsfile2 to upgrade pip --- Jenkinsfile2 | 2 ++ cmake/taostools_CMakeLists.txt.in | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 423169c007..75c408186f 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -309,6 +309,7 @@ def pre_test_build_win() { ''' bat ''' cd %WIN_CONNECTOR_ROOT% + python.exe -m pip install --upgrade pip python -m pip install . xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ''' @@ -327,6 +328,7 @@ def run_win_test() { bat ''' echo "windows test ..." cd %WIN_CONNECTOR_ROOT% + python.exe -m pip install --upgrade pip python -m pip install . xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ls -l C:\\Windows\\System32\\taos.dll diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 4f08656ac8..ad09ce24cf 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 2d68404 + GIT_TAG 57bdfbf SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 11ae805c7f3f4132adf1d782b1704f2d95b4d6d5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 11 Aug 2022 11:18:06 +0800 Subject: [PATCH 151/165] fix: fix hash value search issue --- source/libs/catalog/src/ctgUtil.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 5b3ae013bd..7b81343358 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -896,6 +896,18 @@ int32_t ctgHashValueComp(void const *lp, void const *rp) { return 0; } +int ctgVgInfoComp(const void* lp, const void* rp) { + SVgroupInfo *pLeft = *(SVgroupInfo **)lp; + SVgroupInfo *pRight = *(SVgroupInfo **)rp; + if (pLeft->hashBegin < pRight->hashBegin) { + return -1; + } else if (pLeft->hashBegin > pRight->hashBegin) { + return 1; + } + + return 0; +} + int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update) { int32_t code = 0; @@ -948,6 +960,8 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo pIter = taosHashIterate(dbInfo->vgHash, pIter); } + taosArraySort(pVgList, ctgVgInfoComp); + char tbFullName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFullName, "%s.", dbFName); int32_t offset = strlen(tbFullName); @@ -966,6 +980,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo if (NULL == p) { ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, taosHashGetSize(dbInfo->vgHash)); + ASSERT(0); taosArrayDestroy(pVgList); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } From b084ec4eb0fda94c29c5f750cc98f5459f1af785 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Thu, 11 Aug 2022 11:20:57 +0800 Subject: [PATCH 152/165] ci: adapter jenkinsfile for all branches --- Jenkinsfile2 | 90 +++++++++++++--------------------------------------- 1 file changed, 22 insertions(+), 68 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 75c408186f..ed12a0628b 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -38,40 +38,21 @@ def pre_test(){ sh ''' cd ${WK} git reset --hard + git remote prune origin + git fetch cd ${WKC} git reset --hard git clean -fxd + git remote prune origin + git fetch ''' script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - cd ${WK} - git checkout master - cd ${WKC} - git checkout master - ''' - } else if(env.CHANGE_TARGET == '2.0') { - sh ''' - cd ${WK} - git checkout 2.0 - cd ${WKC} - git checkout 2.0 - ''' - } else if(env.CHANGE_TARGET == '3.0') { - sh ''' - cd ${WK} - git checkout 3.0 - cd ${WKC} - git checkout 3.0 - ''' - } else { - sh ''' - cd ${WK} - git checkout develop - cd ${WKC} - git checkout develop - ''' - } + sh ''' + cd ${WK} + git checkout ''' + env.CHANGE_TARGET + ''' + cd ${WKC} + git checkout ''' + env.CHANGE_TARGET + ''' + ''' } if (env.CHANGE_URL =~ /\/TDengine\//) { sh ''' @@ -169,49 +150,24 @@ def pre_test_win(){ bat ''' cd %WIN_INTERNAL_ROOT% git reset --hard + git remote prune origin + git fetch ''' bat ''' cd %WIN_COMMUNITY_ROOT% git reset --hard + git remote prune origin + git fetch ''' script { - if (env.CHANGE_TARGET == 'master') { - bat ''' - cd %WIN_INTERNAL_ROOT% - git checkout master - ''' - bat ''' - cd %WIN_COMMUNITY_ROOT% - git checkout master - ''' - } else if(env.CHANGE_TARGET == '2.0') { - bat ''' - cd %WIN_INTERNAL_ROOT% - git checkout 2.0 - ''' - bat ''' - cd %WIN_COMMUNITY_ROOT% - git checkout 2.0 - ''' - } else if(env.CHANGE_TARGET == '3.0') { - bat ''' - cd %WIN_INTERNAL_ROOT% - git checkout 3.0 - ''' - bat ''' - cd %WIN_COMMUNITY_ROOT% - git checkout 3.0 - ''' - } else { - bat ''' - cd %WIN_INTERNAL_ROOT% - git checkout develop - ''' - bat ''' - cd %WIN_COMMUNITY_ROOT% - git checkout develop - ''' - } + bat ''' + cd %WIN_INTERNAL_ROOT% + git checkout ''' + env.CHANGE_TARGET + ''' + ''' + bat ''' + cd %WIN_COMMUNITY_ROOT% + git checkout ''' + env.CHANGE_TARGET + ''' + ''' } script { if (env.CHANGE_URL =~ /\/TDengine\//) { @@ -309,7 +265,6 @@ def pre_test_build_win() { ''' bat ''' cd %WIN_CONNECTOR_ROOT% - python.exe -m pip install --upgrade pip python -m pip install . xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ''' @@ -328,7 +283,6 @@ def run_win_test() { bat ''' echo "windows test ..." cd %WIN_CONNECTOR_ROOT% - python.exe -m pip install --upgrade pip python -m pip install . xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ls -l C:\\Windows\\System32\\taos.dll From 7d015bd7504086bfef8c91742998188be3eea387 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 11 Aug 2022 11:28:55 +0800 Subject: [PATCH 153/165] docs: remove continous query from connector doc (#15968) * docs: update connector for 3.0 * docs: remove continous query from connector doc --- docs/zh/14-reference/03-connector/03-connector.mdx | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/zh/14-reference/03-connector/03-connector.mdx b/docs/zh/14-reference/03-connector/03-connector.mdx index e18759e600..00b76af867 100644 --- a/docs/zh/14-reference/03-connector/03-connector.mdx +++ b/docs/zh/14-reference/03-connector/03-connector.mdx @@ -49,7 +49,6 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | -------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | | **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **连续查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | ** TMQ ** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **Schemaless** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | From 253257e422e3a975d61fc67d492a5a115d8d55bb Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 11 Aug 2022 11:41:06 +0800 Subject: [PATCH 154/165] doc: refine some pages --- docs/zh/05-get-started/01-docker.md | 4 ++++ docs/zh/12-taos-sql/22-meta.md | 4 ++-- docs/zh/12-taos-sql/23-perf.md | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index 741f6dfeab..8e2b7cc554 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -63,6 +63,10 @@ taos> taosAdapter 是 TDengine 中提供 REST 服务的组件。下面这条命令会在容器中同时启动 `taosd` 和 `taosadapter` 两个服务组件。默认 Docker 镜像同时启动 TDengine 后台服务 taosd 和 taosAdatper。 +```shell +docker run -d --name tdengine -p 6041:6041 tdengine/tdengine +``` + 可以在宿主机使用 curl 通过 RESTful 端口访问 Docker 容器内的 TDengine server。 ``` diff --git a/docs/zh/12-taos-sql/22-meta.md b/docs/zh/12-taos-sql/22-meta.md index 1e17870685..8139b2fc55 100644 --- a/docs/zh/12-taos-sql/22-meta.md +++ b/docs/zh/12-taos-sql/22-meta.md @@ -1,6 +1,6 @@ --- -sidebar_label: 元数据库 -title: 元数据库 +sidebar_label: 元数据 +title: 存储元数据的 Information_Schema 数据库 --- TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数据库元数据、数据库系统信息和状态的访问，例如数据库或表的名称，当前执行的 SQL 语句等。该数据库存储有关 TDengine 维护的所有其他数据库的信息。它包含多个只读表。实际上，这些表都是视图，而不是基表，因此没有与它们关联的文件。所以对这些表只能查询，不能进行 INSERT 等写入操作。`INFORMATION_SCHEMA` 数据库旨在以一种更一致的方式来提供对 TDengine 支持的各种 SHOW 语句（如 SHOW TABLES、SHOW DATABASES）所提供的信息的访问。与 SHOW 语句相比，使用 SELECT ... FROM INFORMATION_SCHEMA.tablename 具有以下优点： diff --git a/docs/zh/12-taos-sql/23-perf.md b/docs/zh/12-taos-sql/23-perf.md index 4c7660e829..3ff8e3f72c 100644 --- a/docs/zh/12-taos-sql/23-perf.md +++ b/docs/zh/12-taos-sql/23-perf.md @@ -1,6 +1,6 @@ --- -sidebar_label: 性能数据库 -title: 性能数据库 +sidebar_label: 统计数据 +title: 存储统计数据的 Performance_Schema 数据库 --- TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其中存储了与性能有关的统计数据。本节详细介绍其中的表和详细的表结构。 From c191f44b1426b5bb81c139c4e75851a3d6c7bebd Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 11 Aug 2022 12:01:20 +0800 Subject: [PATCH 155/165] feat(stream): session window trigger delete --- include/common/tcommon.h | 21 +++++--- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/sma/smaTimeRange.c | 5 +- source/dnode/vnode/src/tq/tqSink.c | 62 ++++++++++++++++++++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 3 +- source/libs/executor/inc/executorimpl.h | 7 --- 6 files changed, 76 insertions(+), 24 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index be18ef1fc0..e04d9d5e86 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -103,12 +103,12 @@ typedef struct SDataBlockInfo { int16_t hasVarCol; uint32_t capacity; // TODO: optimize and remove following - int64_t version; // used for stream, and need serialization - int64_t ts; // used for stream, and need serialization - int32_t childId; // used for stream, do not serialize - EStreamType type; // used for stream, do not serialize - STimeWindow calWin; // used for stream, do not serialize - TSKEY watermark;// used for stream + int64_t version; // used for stream, and need serialization + int64_t ts; // used for stream, and need serialization + int32_t childId; // used for stream, do not serialize + EStreamType type; // used for stream, do not serialize + STimeWindow calWin; // used for stream, do not serialize + TSKEY watermark; // used for stream } SDataBlockInfo; typedef struct SSDataBlock { @@ -268,6 +268,15 @@ typedef struct SSortExecInfo { int32_t readBytes; // read io bytes } SSortExecInfo; +// stream special block column + +#define START_TS_COLUMN_INDEX 0 +#define END_TS_COLUMN_INDEX 1 +#define UID_COLUMN_INDEX 2 +#define GROUPID_COLUMN_INDEX 3 +#define CALCULATE_START_TS_COLUMN_INDEX 4 +#define CALCULATE_END_TS_COLUMN_INDEX 5 + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index ffc1966733..43bb92ec23 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -171,7 +171,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list); -SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, +SSubmitReq* tdBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, const char* stbFullName, int32_t vgId, SBatchDeleteReq* pDeleteReq); // sma diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 8da397f0c3..f46d9dc29c 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -201,8 +201,9 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { } SBatchDeleteReq deleteReq; - SSubmitReq *pSubmitReq = tdBlockToSubmit((const SArray *)msg, pTsmaStat->pTSchema, true, pTsmaStat->pTSma->dstTbUid, - pTsmaStat->pTSma->dstTbName, pTsmaStat->pTSma->dstVgId, &deleteReq); + SSubmitReq *pSubmitReq = + tdBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, true, pTsmaStat->pTSma->dstTbUid, + pTsmaStat->pTSma->dstTbName, pTsmaStat->pTSma->dstVgId, &deleteReq); if (!pSubmitReq) { smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 2b3283a395..f37574a064 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -13,10 +13,44 @@ * along with this program. If not, see . */ +#include "tcommon.h" +#include "tmsg.h" #include "tq.h" -SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId, SBatchDeleteReq* deleteReq) { +int32_t tdBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBlock* pDataBlock, + SBatchDeleteReq* deleteReq) { + ASSERT(pDataBlock->info.type == STREAM_DELETE_RESULT); + int32_t totRow = pDataBlock->info.rows; + SColumnInfoData* pTsCol = taosArrayGet(pDataBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pGidCol = taosArrayGet(pDataBlock->pDataBlock, GROUPID_COLUMN_INDEX); + for (int32_t row = 0; row < totRow; row++) { + int64_t ts = *(int64_t*)colDataGetData(pTsCol, row); + /*int64_t groupId = *(int64_t*)colDataGetData(pGidCol, row);*/ + int64_t groupId = 0; + char* name = buildCtbNameByGroupId(stbFullName, groupId); + tqDebug("delete msg: groupId :%ld, name: %s", groupId, name); + SMetaReader mr = {0}; + metaReaderInit(&mr, pVnode->pMeta, 0); + if (metaGetTableEntryByName(&mr, name) < 0) { + metaReaderClear(&mr); + taosMemoryFree(name); + return -1; + } + + int64_t uid = mr.me.uid; + metaReaderClear(&mr); + taosMemoryFree(name); + SSingleDeleteReq req = { + .ts = ts, + .uid = uid, + }; + taosArrayPush(deleteReq->deleteReqs, &req); + } + return 0; +} + +SSubmitReq* tdBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, bool createTb, + int64_t suid, const char* stbFullName, int32_t vgId, SBatchDeleteReq* pDeleteReq) { SSubmitReq* ret = NULL; SArray* schemaReqs = NULL; SArray* schemaReqSz = NULL; @@ -33,9 +67,13 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo schemaReqSz = taosArrayInit(sz, sizeof(int32_t)); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_DATA) { - // + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + int32_t padding1 = 0; + void* padding2 = taosMemoryMalloc(1); + taosArrayPush(schemaReqSz, &padding1); + taosArrayPush(schemaReqs, &padding2); } + STagVal tagVal = { .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, .type = TSDB_DATA_TYPE_UBIGINT, @@ -97,7 +135,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo int32_t cap = sizeof(SSubmitReq); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - int32_t rows = pDataBlock->info.rows; + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + continue; + } + int32_t rows = pDataBlock->info.rows; // TODO min int32_t rowSize = pDataBlock->info.rowSize; int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); @@ -119,6 +160,11 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + pDeleteReq->suid = suid; + tdBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); + continue; + } blkHead->numOfRows = htonl(pDataBlock->info.rows); blkHead->sversion = htonl(pTSchema->version); @@ -187,7 +233,7 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { ASSERT(pTask->tbSink.pTSchema); deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); - SSubmitReq* pReq = tdBlockToSubmit(pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid, + SSubmitReq* pReq = tdBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid, pTask->tbSink.stbFullName, pVnode->config.vgId, &deleteReq); tqDebug("vgId:%d, task %d convert blocks over, put into write-queue", TD_VID(pVnode), pTask->taskId); @@ -200,12 +246,14 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { ASSERT(0); } SEncoder encoder; - void* buf = taosMemoryCalloc(1, len + sizeof(SMsgHead)); + void* buf = rpcMallocCont(len + sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&encoder, abuf, len); tEncodeSBatchDeleteReq(&encoder, &deleteReq); tEncoderClear(&encoder); + ((SMsgHead*)buf)->vgId = pVnode->config.vgId; + if (taosArrayGetSize(deleteReq.deleteReqs) != 0) { SRpcMsg msg = { .msgType = TDMT_VND_BATCH_DEL, diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 1f13bde0c1..ecff58f3b1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -145,7 +145,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t len; int32_t ret; - vTrace("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); pVnode->state.applied = version; @@ -1071,6 +1071,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void // TODO } } + taosArrayDestroy(deleteReq.deleteReqs); return 0; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8439cf700d..9faa1cec3e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -52,13 +52,6 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData ? 1 : 0) -#define START_TS_COLUMN_INDEX 0 -#define END_TS_COLUMN_INDEX 1 -#define UID_COLUMN_INDEX 2 -#define GROUPID_COLUMN_INDEX 3 -#define CALCULATE_START_TS_COLUMN_INDEX 4 -#define CALCULATE_END_TS_COLUMN_INDEX 5 - enum { // when this task starts to execute, this status will set TASK_NOT_COMPLETED = 0x1u, From b6057ee2e183baf0ee66fa1f9f1aad53d78d10dd Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Thu, 11 Aug 2022 13:27:04 +0800 Subject: [PATCH 156/165] Update 23-perf.md --- docs/zh/12-taos-sql/23-perf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/23-perf.md b/docs/zh/12-taos-sql/23-perf.md index 3ff8e3f72c..1ff7da6c71 100644 --- a/docs/zh/12-taos-sql/23-perf.md +++ b/docs/zh/12-taos-sql/23-perf.md @@ -3,7 +3,7 @@ sidebar_label: 统计数据 title: 存储统计数据的 Performance_Schema 数据库 --- -TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其中存储了与性能有关的统计数据。本节详细介绍其中的表和详细的表结构。 +TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其中存储了与性能有关的统计数据。本节详细介绍其中的表和表结构。 ## PERF_APP From b8a3654cd625127d0f09cb67daf823f5f2cd02fa Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 11 Aug 2022 13:37:32 +0800 Subject: [PATCH 157/165] enh(stream): show stream --- include/libs/stream/tstream.h | 2 ++ source/common/src/systable.c | 4 +-- source/dnode/mnode/impl/src/mndStream.c | 34 +++++++++++++++++++++++-- source/dnode/vnode/src/tq/tqSink.c | 2 +- tests/script/jenkins/basic.txt | 4 +-- 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 32dc9e1866..f51c37ed47 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -34,6 +34,8 @@ typedef struct SStreamTask SStreamTask; enum { STREAM_STATUS__NORMAL = 0, + STREAM_STATUS__STOP, + STREAM_STATUS__FAILED, STREAM_STATUS__RECOVER, }; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index be76a1b453..16681fb705 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -135,12 +135,12 @@ static const SSysDbTableSchema streamSchema[] = { {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "source_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "target_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "target_table", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "watermark", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "trigger", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "trigger", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; static const SSysDbTableSchema userTblsSchema[] = { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b4af39e467..8c453e0c88 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -197,6 +197,30 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { sdbRelease(pSdb, pStream); } +static void mndShowStreamStatus(char *dst, SStreamObj *pStream) { + int8_t status = atomic_load_8(&pStream->status); + if (status == STREAM_STATUS__NORMAL) { + strcpy(dst, "normal"); + } else if (status == STREAM_STATUS__STOP) { + strcpy(dst, "stop"); + } else if (status == STREAM_STATUS__FAILED) { + strcpy(dst, "failed"); + } else if (status == STREAM_STATUS__RECOVER) { + strcpy(dst, "recover"); + } +} + +static void mndShowStreamTrigger(char *dst, SStreamObj *pStream) { + int8_t trigger = pStream->trigger; + if (trigger == STREAM_TRIGGER_AT_ONCE) { + strcpy(dst, "at once"); + } else if (trigger == STREAM_TRIGGER_WINDOW_CLOSE) { + strcpy(dst, "window close"); + } else if (trigger == STREAM_TRIGGER_MAX_DELAY) { + strcpy(dst, "max delay"); + } +} + static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->sourceDB[0] == 0 || pCreate->targetStbFullName[0] == 0) { @@ -926,8 +950,11 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)sql, false); + char status[20 + VARSTR_HEADER_SIZE] = {0}; + mndShowStreamStatus(&status[VARSTR_HEADER_SIZE], pStream); + varDataSetLen(status, strlen(varDataVal(status))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->status, true); + colDataAppend(pColInfo, numOfRows, (const char *)&status, false); char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; tNameFromString(&n, pStream->sourceDb, T_NAME_ACCT | T_NAME_DB); @@ -958,8 +985,11 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); + char trigger[20 + VARSTR_HEADER_SIZE] = {0}; + mndShowStreamTrigger(&trigger[VARSTR_HEADER_SIZE], pStream); + varDataSetLen(trigger, strlen(varDataVal(trigger))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->trigger, false); + colDataAppend(pColInfo, numOfRows, (const char *)&trigger, false); numOfRows++; sdbRelease(pSdb, pStream); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index f37574a064..503d086e1d 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -28,7 +28,7 @@ int32_t tdBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl /*int64_t groupId = *(int64_t*)colDataGetData(pGidCol, row);*/ int64_t groupId = 0; char* name = buildCtbNameByGroupId(stbFullName, groupId); - tqDebug("delete msg: groupId :%ld, name: %s", groupId, name); + tqDebug("stream delete msg: groupId :%ld, name: %s", groupId, name); SMetaReader mr = {0}; metaReaderInit(&mr, pVnode->pMeta, 0); if (metaGetTableEntryByName(&mr, name) < 0) { diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0bb24826a3..cf7e2d63f0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -237,8 +237,8 @@ ./test.sh -f tsim/stream/distributeInterval0.sim ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ./test.sh -f tsim/stream/distributeSession0.sim -./test.sh -f tsim/stream/session0.sim -./test.sh -f tsim/stream/session1.sim +#./test.sh -f tsim/stream/session0.sim +#./test.sh -f tsim/stream/session1.sim ./test.sh -f tsim/stream/state0.sim ./test.sh -f tsim/stream/triggerInterval0.sim ./test.sh -f tsim/stream/triggerSession0.sim From 98f4d0c0dde9b3b6b4f249b301b0689b895b62dd Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 11 Aug 2022 13:57:23 +0800 Subject: [PATCH 158/165] test: add interp test cases --- tests/system-test/2-query/sample.py | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index f09265c300..cb74c33d92 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -335,7 +335,7 @@ class TDTestCase: # case11 = {"alias": ", st1"} # self.checksample(**case11) tdSql.query("select sample( c1 , 1 ) , st1 from t1") - + # case12 = {"alias": ", c1"} # self.checksample(**case12) @@ -497,7 +497,7 @@ class TDTestCase: # tdSql.query(" select sample(c1 , 1) + 2 from t1 ") err41 = {"alias": "+ avg(c1)"} # self.checksample(**err41) # mix with arithmetic 2 - + # err42 = {"alias": ", c1"} # self.checksample(**err42) tdSql.query("select sample( c1 , 1 ) , c1 from t1") @@ -605,14 +605,14 @@ class TDTestCase: tdSql.execute(f"create table tt{i} using stb2 tags({i})") pass - + def check_sample(self , sample_query , origin_query ): tdSql.query(origin_query) origin_datas = tdSql.queryResult - + tdSql.query(sample_query) sample_datas = tdSql.queryResult @@ -620,7 +620,7 @@ class TDTestCase: for ind , sample_data in enumerate(sample_datas): if sample_data not in origin_datas: status = False - + if status: tdLog.info(" sample data is in datas groups ,successed sql is : %s" % sample_query ) else: @@ -637,7 +637,7 @@ class TDTestCase: tags (t1 int) ''' ) - + tdSql.execute( ''' create table t1 @@ -689,7 +689,7 @@ class TDTestCase: tdSql.error(" select sample(c1,ts) from t1 ") tdSql.error(" select sample(c1,false) from t1 ") tdSql.query(" select sample(123,1) from t1 ") - + tdSql.query(" select sample(c1,2) from t1 ") tdSql.checkRows(2) tdSql.query(" select sample(c1,10) from t1 ") @@ -704,10 +704,10 @@ class TDTestCase: tdSql.checkRows(9) tdSql.error(" select sample(c1,-1) from t1 ") - # bug need fix + # bug need fix # tdSql.query("select sample(c1 ,2) , 123 from stb1;") - # all type support + # all type support tdSql.query(" select sample(c1 , 20 ) from ct4 ") tdSql.checkRows(9) @@ -761,7 +761,7 @@ class TDTestCase: self.check_sample("select sample( c1 ,3 ) from t1 where c1 between 1 and 10" ,"select c1 from t1 where c1 between 1 and 10") - # join + # join tdSql.query("select sample( ct4.c1 , 1 ) from ct1, ct4 where ct4.ts=ct1.ts") @@ -772,22 +772,22 @@ class TDTestCase: self.check_sample("select sample(c1,2) from stb1 partition by tbname" , "select c1 from stb1 partition by tbname") - # nest query + # nest query # tdSql.query("select sample(c1,2) from (select c1 from t1); ") # tdSql.checkRows(2) - # union all + # union all tdSql.query("select sample(c1,2) from t1 union all select sample(c1,3) from t1") tdSql.checkRows(5) # fill interval - # not support mix with other function + # not support mix with other function tdSql.error("select top(c1,2) , sample(c1,2) from ct1") tdSql.error("select max(c1) , sample(c1,2) from ct1") tdSql.query("select c1 , sample(c1,2) from ct1") - # bug for mix with scalar + # bug for mix with scalar tdSql.query("select 123 , sample(c1,100) from ct1") tdSql.query("select sample(c1,100)+2 from ct1") tdSql.query("select abs(sample(c1,100)) from ct1") @@ -864,13 +864,13 @@ class TDTestCase: for i in range(2000): ts = self.ts+i*10 tdSql.execute(f"insert into sub_tb values({ts} ,{i})") - + tdSql.query("select count(*) from st") tdSql.checkData(0,0,2000) tdSql.query("select sample(c1 ,1000) from st") tdSql.checkRows(1000) - # bug need fix + # bug need fix tdSql.query("select c1 ,t1, sample(c1,2) from db.stb1 partition by c1 ") tdSql.query("select sample(c1,2) from db.stb1 partition by c1 ") # tdSql.query("select c1 ,ind, sample(c1,2) from sample_db.st partition by c1 ") From a270c989d2e51861a2c25def7776b009c246f21c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 11 Aug 2022 13:57:23 +0800 Subject: [PATCH 159/165] test: add interp test cases --- tests/system-test/fulltest.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 7941e76df6..cdc0085cbc 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -96,6 +96,8 @@ python3 ./test.py -f 2-query/distribute_agg_stddev.py python3 ./test.py -f 2-query/distribute_agg_stddev.py -R python3 ./test.py -f 2-query/distribute_agg_sum.py python3 ./test.py -f 2-query/distribute_agg_sum.py -R +python3 ./test.py -f 2-query/interp.py +python3 ./test.py -f 2-query/interp.py -R @@ -338,6 +340,7 @@ python3 ./test.py -f 2-query/arcsin.py -Q 2 python3 ./test.py -f 2-query/arccos.py -Q 2 python3 ./test.py -f 2-query/arctan.py -Q 2 python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 +python3 ./test.py -f 2-query/interp.py -Q 2 # python3 ./test.py -f 2-query/nestedQuery.py -Q 2 # python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 @@ -457,3 +460,4 @@ python3 ./test.py -f 2-query/max_partition.py -Q 3 python3 ./test.py -f 2-query/last_row.py -Q 3 python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 python3 ./test.py -f 2-query/sml.py -Q 3 +python3 ./test.py -f 2-query/interp.py -Q 3 From 67eb7ec4080c8cbf0cb309407017b4657279615c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 11 Aug 2022 13:57:23 +0800 Subject: [PATCH 160/165] test: add interp test cases --- tests/system-test/2-query/interp.py | 559 ++++++++++++++++++++++++++++ 1 file changed, 559 insertions(+) create mode 100644 tests/system-test/2-query/interp.py diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py new file mode 100644 index 0000000000..9348a8ca8f --- /dev/null +++ b/tests/system-test/2-query/interp.py @@ -0,0 +1,559 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): + dbname = "db" + tbname = "tb" + + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + + tdSql.execute( + f'''create table if not exists {dbname}.{tbname} + (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10)) + ''' + ) + + tdLog.printNoPrefix("==========step2:insert data") + + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:05', 5, 5, 5, 5, 5.0, 5.0, true, 'varchar', 'nchar')") + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:10', 10, 10, 10, 10, 10.0, 10.0, true, 'varchar', 'nchar')") + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:15', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar')") + + tdLog.printNoPrefix("==========step3:fill null") + + ## {. . .} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(null)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, None) + + ## {} ... + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(null)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + + ## {.}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:03', '2020-02-01 00:00:07') every(1s) fill(null)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + ## .{}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:06', '2020-02-01 00:00:09') every(1s) fill(null)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + + ## .{.}. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(1s) fill(null)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + ## ..{.} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(null)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + ## ... {} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(null)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + + tdLog.printNoPrefix("==========step4:fill value") + + ## {. . .} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 1) + tdSql.checkData(9, 0, 1) + tdSql.checkData(10, 0, 1) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, 1) + + ## {} ... + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(value, 1)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + + ## {.}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:03', '2020-02-01 00:00:07') every(1s) fill(value, 1)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + + ## .{}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:06', '2020-02-01 00:00:09') every(1s) fill(value, 1)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + + ## .{.}. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(1s) fill(value, 1)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + + ## ..{.} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(value, 1)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + + ## ... {} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + + tdLog.printNoPrefix("==========step5:fill prev") + + ## {. . .} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(prev)") + tdSql.checkRows(12) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 5) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 10) + tdSql.checkData(8, 0, 10) + tdSql.checkData(9, 0, 10) + tdSql.checkData(10, 0, 15) + tdSql.checkData(11, 0, 15) + + ## {} ... + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(prev)") + tdSql.checkRows(0) + + ## {.}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:03', '2020-02-01 00:00:07') every(1s) fill(prev)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + + ## .{}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:06', '2020-02-01 00:00:09') every(1s) fill(prev)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + + ## .{.}. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(1s) fill(prev)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 10) + tdSql.checkData(4, 0, 10) + + ## ..{.} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(prev)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 10) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + + ## ... {} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(prev)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 15) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + + tdLog.printNoPrefix("==========step6:fill next") + + ## {. . .} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") + tdSql.checkRows(12) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 10) + tdSql.checkData(4, 0, 10) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 15) + tdSql.checkData(8, 0, 15) + tdSql.checkData(9, 0, 15) + tdSql.checkData(10, 0, 15) + tdSql.checkData(11, 0, 15) + + ## {} ... + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(next)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + + ## {.}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:03', '2020-02-01 00:00:07') every(1s) fill(next)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 10) + tdSql.checkData(4, 0, 10) + + ## .{}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:06', '2020-02-01 00:00:09') every(1s) fill(next)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 10) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 10) + + ## .{.}. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(1s) fill(next)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 10) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + + ## ..{.} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(next)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, 15) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 15) + + ## ... {} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(next)") + tdSql.checkRows(0) + + + tdLog.printNoPrefix("==========step7:fill linear") + + ## {. . .} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 7) + tdSql.checkData(3, 0, 8) + tdSql.checkData(4, 0, 9) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, 11) + tdSql.checkData(7, 0, 12) + tdSql.checkData(8, 0, 13) + tdSql.checkData(9, 0, 14) + tdSql.checkData(10, 0, 15) + + ## {} ... + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(linear)") + tdSql.checkRows(0) + + ## {.}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:03', '2020-02-01 00:00:07') every(1s) fill(linear)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 7) + + ## .{}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:06', '2020-02-01 00:00:09') every(1s) fill(linear)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 6) + tdSql.checkData(1, 0, 7) + tdSql.checkData(2, 0, 8) + tdSql.checkData(3, 0, 9) + + ## .{.}. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(1s) fill(linear)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 8) + tdSql.checkData(1, 0, 9) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 11) + tdSql.checkData(4, 0, 12) + + ## ..{.} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(linear)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, 13) + tdSql.checkData(1, 0, 14) + tdSql.checkData(2, 0, 15) + + ## ... {} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(linear)") + tdSql.checkRows(0) + + + tdLog.printNoPrefix("==========step8:test intra block interpolation") + tdSql.execute(f"drop database {dbname}"); + + tdSql.prepare() + + tdSql.execute(f"create table if not exists {dbname}.{tbname} (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10))") + + # set two data point has 10 days interval will be stored in different datablocks + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:05', 5, 5, 5, 5, 5.0, 5.0, true, 'varchar', 'nchar')") + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-11 00:00:05', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar')") + + tdSql.execute(f"flush database {dbname}"); + + # test fill null + + ## | {. | | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-11 00:00:05') every(1d) fill(null)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 15) + + ## | . | {} | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-03 00:00:05', '2020-02-07 00:00:05') every(1d) fill(null)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + ## | {. | } | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-01-31 00:00:05', '2020-02-05 00:00:05') every(1d) fill(null)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + + ## | . | { | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + + + # test fill value + + ## | {. | | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-11 00:00:05') every(1d) fill(value, 1)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 1) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 1) + tdSql.checkData(9, 0, 1) + tdSql.checkData(10, 0, 15) + + ## | . | {} | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-03 00:00:05', '2020-02-07 00:00:05') every(1d) fill(value, 1)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + + ## | {. | } | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-01-31 00:00:05', '2020-02-05 00:00:05') every(1d) fill(value, 1)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + + ## | . | { | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(value, 1)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + + + # test fill prev + + ## | {. | | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-11 00:00:05') every(1d) fill(prev)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 5) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 5) + tdSql.checkData(7, 0, 5) + tdSql.checkData(8, 0, 5) + tdSql.checkData(9, 0, 5) + tdSql.checkData(10, 0, 15) + + ## | . | {} | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-03 00:00:05', '2020-02-07 00:00:05') every(1d) fill(prev)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 5) + + ## | {. | } | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-01-31 00:00:05', '2020-02-05 00:00:05') every(1d) fill(prev)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 5) + + ## | . | { | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(prev)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + tdSql.checkData(5, 0, 15) + + # test fill next + + ## | {. | | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-11 00:00:05') every(1d) fill(next)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + tdSql.checkData(5, 0, 15) + tdSql.checkData(6, 0, 15) + tdSql.checkData(7, 0, 15) + tdSql.checkData(8, 0, 15) + tdSql.checkData(9, 0, 15) + tdSql.checkData(10, 0, 15) + + ## | . | {} | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-03 00:00:05', '2020-02-07 00:00:05') every(1d) fill(next)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 15) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + + ## | {. | } | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-01-31 00:00:05', '2020-02-05 00:00:05') every(1d) fill(next)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + tdSql.checkData(5, 0, 15) + + ## | . | { | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(next)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 15) + tdSql.checkData(1, 0, 15) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 7836b581a7d36ee48b7e5cac0dbbeeac96e2a8b8 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 11 Aug 2022 14:25:16 +0800 Subject: [PATCH 161/165] feat(stream): delete result --- source/libs/executor/inc/executorimpl.h | 3 +-- source/libs/executor/src/timewindowoperator.c | 27 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 75db151f85..839c9e61ed 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -694,6 +694,7 @@ typedef struct SSessionAggOperatorInfo { typedef struct SResultWindowInfo { SResultRowPosition pos; STimeWindow win; + uint64_t groupId; bool isOutput; bool isClosed; } SResultWindowInfo; @@ -1008,8 +1009,6 @@ SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY star SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex); bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); -int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, - TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 2b67f2bf01..6020794429 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3734,7 +3734,7 @@ SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY star return insertNewSessionWindow(pWinInfos, startTs, index + 1); } -int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, +int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId,int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted) { for (int32_t i = start; i < rows; ++i) { if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) { @@ -3742,7 +3742,8 @@ int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TS } if (pWinInfo->win.skey > pStartTs[i]) { if (pStDeleted && pWinInfo->isOutput) { - taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY)); + SWinRes res = {.ts = pWinInfo->win.skey, .groupId = groupId}; + taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinRes)); pWinInfo->isOutput = false; } pWinInfo->win.skey = pStartTs[i]; @@ -3861,7 +3862,8 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo); taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition)); if (pWinInfo->isOutput) { - taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY)); + SWinRes res = {.ts = pWinInfo->win.skey, .groupId = groupId}; + taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinRes)); pWinInfo->isOutput = false; } taosArrayRemove(pInfo->streamAggSup.pCurWins, i); @@ -3911,7 +3913,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData int32_t winIndex = 0; SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex); winRows = - updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted); + updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, groupId, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted); code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -3960,6 +3962,7 @@ static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc } deleteWindow(pAggSup->pCurWins, winIndex, fp); if (result) { + pCurWin->groupId = gpDatas[i]; taosArrayPush(result, pCurWin); } } @@ -3980,7 +3983,7 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, step = 1; continue; } - step = updateSessionWindowInfo(pCurWin, tsCols, NULL, pBlock->info.rows, i, gap, NULL); + step = updateSessionWindowInfo(pCurWin, tsCols, NULL, 0, pBlock->info.rows, i, gap, NULL); ASSERT(isInWindow(pCurWin, tsCols[i], gap)); doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput); if (result) { @@ -4017,12 +4020,11 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It blockDataEnsureCapacity(pBlock, size); size_t keyLen = 0; while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false); - for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); i++) { - pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - colDataAppendNULL(pColInfoData, pBlock->info.rows); - } + SWinRes* res = *Ite; + SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + colDataAppend(pTsCol, pBlock->info.rows, (const char*)&res->ts, false); + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + colDataAppend(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false); pBlock->info.rows += 1; if (pBlock->info.rows + 1 >= pBlock->info.capacity) { break; @@ -4149,7 +4151,8 @@ static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) { int32_t size = taosArrayGetSize(pResWins); for (int32_t i = 0; i < size; i++) { SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i); - taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY)); + SWinRes res = {.ts = pWinInfo->win.skey, .groupId = pWinInfo->groupId}; + taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinRes)); } } From 80a01235f2b33b0e39c33b7cc1af5612cc742d37 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 11 Aug 2022 14:37:43 +0800 Subject: [PATCH 162/165] doc: change 3.0 doc --- README-CN.md | 14 +------------ README.md | 16 ++------------- cmake/cmake.install | 4 +++- .../03-connector/03-connector.mdx | 1 - .../14-reference/09-support-platform/index.md | 20 +++++++++---------- .../03-connector/03-connector.mdx | 1 - .../14-reference/09-support-platform/index.md | 20 +++++++++---------- packaging/tools/install.sh | 19 ++++++++++++------ source/dnode/mgmt/node_mgmt/src/dmEnv.c | 14 +++++++++++++ tools/shell/src/shellArguments.c | 6 +++--- tools/shell/src/shellEngine.c | 2 +- 11 files changed, 57 insertions(+), 60 deletions(-) diff --git a/README-CN.md b/README-CN.md index b2ae777837..d3f9b47a0f 100644 --- a/README-CN.md +++ b/README-CN.md @@ -188,7 +188,7 @@ apt install autoconf cmake .. -DJEMALLOC_ENABLED=true ``` -在 X86-64、X86、arm64、arm32 和 mips64 平台上，TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型，如 aarch64 或 aarch32 等。 +在 X86-64、X86、arm64 平台上，TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型，如 aarch64 等。 aarch64： @@ -196,18 +196,6 @@ aarch64： cmake .. -DCPUTYPE=aarch64 && cmake --build . ``` -aarch32： - -```bash -cmake .. -DCPUTYPE=aarch32 && cmake --build . -``` - -mips64： - -```bash -cmake .. -DCPUTYPE=mips64 && cmake --build . -``` - ### Windows 系统 如果你使用的是 Visual Studio 2013 版本： diff --git a/README.md b/README.md index 2f38120b34..3ea8a571ef 100644 --- a/README.md +++ b/README.md @@ -205,8 +205,8 @@ apt install autoconf cmake .. -DJEMALLOC_ENABLED=true ``` -TDengine build script can detect the host machine's architecture on X86-64, X86, arm64, arm32 and mips64 platform. -You can also specify CPUTYPE option like aarch64 or aarch32 too if the detection result is not correct: +TDengine build script can detect the host machine's architecture on X86-64, X86, arm64 platform. +You can also specify CPUTYPE option like aarch64 too if the detection result is not correct: aarch64: @@ -214,18 +214,6 @@ aarch64: cmake .. -DCPUTYPE=aarch64 && cmake --build . ``` -aarch32: - -```bash -cmake .. -DCPUTYPE=aarch32 && cmake --build . -``` - -mips64: - -```bash -cmake .. -DCPUTYPE=mips64 && cmake --build . -``` - ### On Windows platform If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". diff --git a/cmake/cmake.install b/cmake/cmake.install index 4b1ccbf6d5..07773d1015 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -22,7 +22,9 @@ ELSEIF (TD_WINDOWS) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosd.exe DESTINATION .) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/udfd.exe DESTINATION .) - INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosBenchmark.exe DESTINATION .) + IF (BUILD_TOOLS) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosBenchmark.exe DESTINATION .) + ENDIF () IF (TD_MVN_INSTALLED) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.38-dist.jar DESTINATION connector/jdbc) diff --git a/docs/en/14-reference/03-connector/03-connector.mdx b/docs/en/14-reference/03-connector/03-connector.mdx index 4468557900..baec6abe38 100644 --- a/docs/en/14-reference/03-connector/03-connector.mdx +++ b/docs/en/14-reference/03-connector/03-connector.mdx @@ -17,7 +17,6 @@ Currently, TDengine's native interface connectors can support platforms such as | **X86 64bit** | **Win32** | ● | ● | ● | ● | ○ | ○ | ● | | **X86 32bit** | **Win32** | ○ | ○ | ○ | ○ | ○ | ○ | ● | | **ARM64** | **Linux** | ● | ● | ● | ● | ○ | ○ | ● | -| **ARM32** | **Linux** | ● | ● | ● | ● | ○ | ○ | ● | | **MIPS** | **Linux** | ○ | ○ | ○ | ○ | ○ | ○ | ○ | Where ● means the official test verification passed, ○ means the unofficial test verification passed, -- means no assurance. diff --git a/docs/en/14-reference/09-support-platform/index.md b/docs/en/14-reference/09-support-platform/index.md index 656344eff4..ee2479e817 100644 --- a/docs/en/14-reference/09-support-platform/index.md +++ b/docs/en/14-reference/09-support-platform/index.md @@ -19,15 +19,15 @@ TDengine's connector can support a wide range of platforms, including X64/X86/AR The comparison matrix is as follows. -| **CPU** | **X64 64bit** | | | **X86 32bit** | **ARM64** | **ARM32** | **MIPS** | **Alpha** | -| ----------- | ------------- | --------- | --------- | ------------- | --------- | --------- | --------- | --------- | -| **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | -| **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | -| **JDBC** | ● | ● | ● | ○ | ● | ● | ● | ● | -| **Python** | ● | ● | ● | ○ | ● | ● | ● | -- | -| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -| **C#** | ● | ● | ○ | ○ | ○ | ○ | ○ | -- | -| **RESTful** | ● | ● | ● | ● | ● | ● | ● | ● | +| **CPU** | **X64 64bit** | | | **X86 32bit** | **ARM64** | **MIPS** | **Alpha** | +| ----------- | ------------- | --------- | --------- | ------------- | --------- | --------- | --------- | +| **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | +| **C/C++** | ● | ● | ● | ○ | ● | ● | ● | +| **JDBC** | ● | ● | ● | ○ | ● | ● | ● | +| **Python** | ● | ● | ● | ○ | ● | ● | -- | +| **Go** | ● | ● | ● | ○ | ● | ○ | -- | +| **NodeJs** | ● | ● | ○ | ○ | ● | ○ | -- | +| **C#** | ● | ● | ○ | ○ | ○ | ○ | -- | +| **RESTful** | ● | ● | ● | ● | ● | ● | ● | Note: ● means the official test is verified, ○ means the unofficial test is verified, -- means not verified. diff --git a/docs/zh/14-reference/03-connector/03-connector.mdx b/docs/zh/14-reference/03-connector/03-connector.mdx index 00b76af867..d681903aed 100644 --- a/docs/zh/14-reference/03-connector/03-connector.mdx +++ b/docs/zh/14-reference/03-connector/03-connector.mdx @@ -17,7 +17,6 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 | **X86 64bit** | **Win32** | ● | ● | ● | ● | ○ | ○ | ● | | **X86 32bit** | **Win32** | ○ | ○ | ○ | ○ | ○ | ○ | ● | | **ARM64** | **Linux** | ● | ● | ● | ● | ○ | ○ | ● | -| **ARM32** | **Linux** | ○ | ○ | ○ | ○ | ○ | ○ | ● | | **MIPS 龙芯** | **Linux** | ○ | ○ | ○ | ○ | ○ | ○ | ○ | | **Alpha 申威** | **Linux** | ○ | ○ | -- | -- | -- | -- | ○ | | **X86 海光** | **Linux** | ○ | ○ | ○ | -- | -- | -- | ○ | diff --git a/docs/zh/14-reference/09-support-platform/index.md b/docs/zh/14-reference/09-support-platform/index.md index e676ab845a..02a3dbdd15 100644 --- a/docs/zh/14-reference/09-support-platform/index.md +++ b/docs/zh/14-reference/09-support-platform/index.md @@ -19,15 +19,15 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" 对照矩阵如下： -| **CPU** | **X64 64bit** | | | **X86 32bit** | **ARM64** | **ARM32** | **MIPS 龙芯** | **Alpha 申威** | **X64 海光** | -| ----------- | ------------- | --------- | --------- | ------------- | --------- | --------- | ------------- | -------------- | ------------ | -| **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** | -| **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | -| **JDBC** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | -| **Python** | ● | ● | ● | ○ | ● | ● | ● | -- | ● | -| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -- | -| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -- | -| **C#** | ● | ● | ○ | ○ | ○ | ○ | ○ | -- | -- | -| **RESTful** | ● | ● | ● | ● | ● | ● | ● | ● | ● | +| **CPU** | **X64 64bit** | | | **X86 32bit** | **ARM64** | **MIPS 龙芯** | **Alpha 申威** | **X64 海光** | +| ----------- | ------------- | --------- | --------- | ------------- | --------- | ------------- | -------------- | ------------ | +| **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | +| **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | +| **JDBC** | ● | ● | ● | ○ | ● | ● | ● | ● | +| **Python** | ● | ● | ● | ○ | ● | ● | -- | ● | +| **Go** | ● | ● | ● | ○ | ● | ○ | -- | -- | +| **NodeJs** | ● | ● | ○ | ○ | ● | ○ | -- | -- | +| **C#** | ● | ● | ○ | ○ | ○ | ○ | -- | -- | +| **RESTful** | ● | ● | ● | ● | ● | ● | ● | ● | 注：● 表示官方测试验证通过，○ 表示非官方测试验证通过，-- 表示未经验证。 diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index beb0718987..eda2b052d1 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -194,6 +194,9 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/${serverName} || : ${csudo}rm -f ${bin_link_dir}/${adapterName} || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : + ${csudo}rm -f ${bin_link_dir}/${demoName} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName} || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : @@ -205,7 +208,6 @@ function install_bin() { [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : - [ -x ${install_main_dir}/bin/${tmqName} ] && ${csudo}ln -s ${install_main_dir}/bin/${tmqName} ${bin_link_dir}/${tmqName} || : [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : @@ -964,12 +966,17 @@ function installProduct() { ## ==============================Main program starts from here============================ serverFqdn=$(hostname) if [ "$verType" == "server" ]; then - # Install server and client - if [ -x ${bin_dir}/${serverName} ]; then - update_flag=1 - updateProduct + # Check default 2.x data file. + if [ -x ${data_dir}/dnode/dnodeCfg.json ]; then + echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of tdengine 2.x, please clear it before installing!\033[0m" else - installProduct + # Install server and client + if [ -x ${bin_dir}/${serverName} ]; then + update_flag=1 + updateProduct + else + installProduct + fi fi elif [ "$verType" == "client" ]; then interactiveFqdn=no diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 5d72ce3b18..81da38076d 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -53,14 +53,27 @@ static bool dmCheckDiskSpace() { osUpdate(); if (!osDataSpaceAvailable()) { dError("free disk size: %f GB, too little, require %f GB at least at least , quit", (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); + terrno = TSDB_CODE_NO_AVAIL_DISK; return false; } if (!osLogSpaceAvailable()) { dError("free disk size: %f GB, too little, require %f GB at least at least, quit", (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); + terrno = TSDB_CODE_NO_AVAIL_DISK; return false; } if (!osTempSpaceAvailable()) { dError("free disk size: %f GB, too little, require %f GB at least at least, quit", (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); + terrno = TSDB_CODE_NO_AVAIL_DISK; + return false; + } + return true; +} + +static bool dmCheckDataDirVersion() { + char checkDataDirJsonFileName[PATH_MAX]; + snprintf(checkDataDirJsonFileName, PATH_MAX, "%s/dnode/dnodeCfg.json", tsDataDir); + if (taosCheckExistFile(checkDataDirJsonFileName)) { + dError("The default data directory %s contains old data of tdengine 2.x, please clear it before running!", tsDataDir); return false; } return true; @@ -68,6 +81,7 @@ static bool dmCheckDiskSpace() { int32_t dmInit(int8_t rtype) { dInfo("start to init dnode env"); + if (!dmCheckDataDirVersion()) return -1; if (!dmCheckDiskSpace()) return -1; if (dmCheckRepeatInit(dmInstance()) != 0) return -1; if (dmInitSystem() != 0) return -1; diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 818cc5425b..93f8b8a23b 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -280,7 +280,7 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { static void shellInitArgs(int argc, char *argv[]) { for (int i = 1; i < argc; i++) { if (strncmp(argv[i], "-p", 2) == 0) { - // printf(shell.info.clientVersion, tsOsName, taos_get_client_info()); + // printf(shell.info.clientVersion, taos_get_client_info()); if (strlen(argv[i]) == 2) { printf("Enter password: "); taosSetConsoleEcho(false); @@ -389,8 +389,8 @@ static int32_t shellCheckArgs() { int32_t shellParseArgs(int32_t argc, char *argv[]) { shellInitArgs(argc, argv); shell.info.clientVersion = - "Welcome to the TDengine shell from %s, Client Version:%s\r\n" - "Copyright (c) 2022 by TAOS Data, Inc. All rights reserved.\r\n\r\n"; + "Welcome to the TDengine Command Line Interface, Client Version:%s\r\n" + "Copyright (c) 2022 by TDengine, all rights reserved.\r\n\r\n"; shell.info.promptHeader = TAOS_CONSOLE_PROMPT_HEADER; shell.info.promptContinue = TAOS_CONSOLE_PROMPT_CONTINUE; shell.info.promptSize = 6; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index f0bda82172..724ac8fbfd 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1001,7 +1001,7 @@ void *shellThreadLoop(void *arg) { } int32_t shellExecute() { - printf(shell.info.clientVersion, shell.info.osname, taos_get_client_info()); + printf(shell.info.clientVersion, taos_get_client_info()); fflush(stdout); SShellArgs *pArgs = &shell.args; From 407b3777ebb3463bb425caa2b687c2453f5c6fd8 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 11 Aug 2022 15:35:40 +0800 Subject: [PATCH 163/165] Update 01-index.md --- docs/zh/01-index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/01-index.md b/docs/zh/01-index.md index 27d4270e4c..771b3f020c 100644 --- a/docs/zh/01-index.md +++ b/docs/zh/01-index.md @@ -4,7 +4,7 @@ sidebar_label: 文档首页 slug: / --- -TDengine是一款开源、[高性能](https://www.taosdata.com/fast)、云原生的时序数据库(Time-Series Database, TSDB), 它专为物联网、工业互联网、金融等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一极简的时序数据处理平台。本文档是 TDengine 用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发者与系统管理员的。 +TDengine是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的时序数据库(Time-Series Database, TSDB), 它专为物联网、工业互联网、金融等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一极简的时序数据处理平台。本文档是 TDengine 用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发者与系统管理员的。 TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用TDengine, 无论如何，请您仔细阅读[基本概念](./concept)一章。 From 50e0ca65723989fbcdf06842b4bb3a59f115efda Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 11 Aug 2022 15:40:58 +0800 Subject: [PATCH 164/165] Update 02-intro.md --- docs/zh/02-intro.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/zh/02-intro.md b/docs/zh/02-intro.md index 91f0376d8c..0c5674b761 100644 --- a/docs/zh/02-intro.md +++ b/docs/zh/02-intro.md @@ -3,7 +3,7 @@ title: 产品简介 toc_max_heading_level: 2 --- -TDengine 是一款开源、高性能、云原生的时序数据库 (Time-Series Database, TSDB)。TDengine 能被广泛运用于物联网、工业互联网、车联网、IT 运维、金融等领域。除核心的时序数据库功能外，TDengine 还提供[缓存](/develop/cache/)、[数据订阅](/develop/subscribe)、[流式计算](/develop/continuous-query)等功能，是一极简的时序数据处理平台，最大程度的减小系统设计的复杂度，降低研发和运营成本。 +TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/tdengine/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database]的时序数据库 (Time-Series Database, TSDB)。TDengine 能被广泛运用于物联网、工业互联网、车联网、IT 运维、金融等领域。除核心的时序数据库功能外，TDengine 还提供[缓存](/develop/cache/)、[数据订阅](/develop/subscribe)、[流式计算](/develop/continuous-query)等功能，是一极简的时序数据处理平台，最大程度的减小系统设计的复杂度，降低研发和运营成本。 本章节介绍TDengine的主要功能、竞争优势、适用场景、与其他数据库的对比测试等等，让大家对TDengine有个整体的了解。 @@ -33,17 +33,17 @@ TDengine的主要功能如下： 由于 TDengine 充分利用了[时序数据特点](https://www.taosdata.com/blog/2019/07/09/105.html)，比如结构化、无需事务、很少删除或更新、写多读少等等，设计了全新的针对时序数据的存储引擎和计算引擎，因此与其他时序数据库相比，TDengine 有以下特点： -- **高性能**：通过创新的存储引擎设计，无论是数据写入还是查询，TDengine 的性能比通用数据库快 10 倍以上，也远超其他时序数据库，存储空间不及通用数据库的1/10。 +- **[高性能](https://www.taosdata.com/tdengine/fast)**：通过创新的存储引擎设计，无论是数据写入还是查询，TDengine 的性能比通用数据库快 10 倍以上，也远超其他时序数据库，存储空间不及通用数据库的1/10。 -- **云原生**：通过原生分布式的设计，充分利用云平台的优势，TDengine 提供了水平扩展能力，具备弹性、韧性和可观测性，支持k8s部署，可运行在公有云、私有云和混合云上。 +- **[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)**：通过原生分布式的设计，充分利用云平台的优势，TDengine 提供了水平扩展能力，具备弹性、韧性和可观测性，支持k8s部署，可运行在公有云、私有云和混合云上。 -- **极简时序数据平台**：TDengine 内建消息队列、缓存、流式计算等功能，应用无需再集成 Kafka/Redis/HBase/Spark 等软件，大幅降低系统的复杂度，降低应用开发和运营成本。 +- **[极简时序数据平台](https://www.taosdata.com/tdengine/simplified_solution_for_time-series_data_processing)**：TDengine 内建消息队列、缓存、流式计算等功能，应用无需再集成 Kafka/Redis/HBase/Spark 等软件，大幅降低系统的复杂度，降低应用开发和运营成本。 -- **分析能力**：支持 SQL，同时为时序数据特有的分析提供SQL扩展。通过超级表、存储计算分离、分区分片、预计算、自定义函数等技术，TDengine 具备强大的分析能力。 +- **[分析能力](https://www.taosdata.com/tdengine/easy_data_analytics)**：支持 SQL，同时为时序数据特有的分析提供SQL扩展。通过超级表、存储计算分离、分区分片、预计算、自定义函数等技术，TDengine 具备强大的分析能力。 -- **简单易用**：无任何依赖，安装、集群几秒搞定；提供REST以及各种语言连接器，与众多第三方工具无缝集成；提供命令行程序，便于管理和即席查询；提供各种运维工具。 +- **[简单易用](https://www.taosdata.com/tdengine/ease_of_use)**：无任何依赖，安装、集群几秒搞定；提供REST以及各种语言连接器，与众多第三方工具无缝集成；提供命令行程序，便于管理和即席查询；提供各种运维工具。 -- **核心开源**：TDengine 的核心代码包括集群功能全部开源，截止到2022年8月1日，全球超过 135.9k 个运行实例，GitHub Star 18.7k，Fork 4.4k，社区活跃。 +- **[核心开源](https://www.taosdata.com/tdengine/open_source_time-series_database)**：TDengine 的核心代码包括集群功能全部开源，截止到2022年8月1日，全球超过 135.9k 个运行实例，GitHub Star 18.7k，Fork 4.4k，社区活跃。 采用 TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。表现在几个方面： From 00d6de5cb9485811d4d52eda7102441f2636d808 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 11 Aug 2022 15:41:44 +0800 Subject: [PATCH 165/165] Update 02-intro.md --- docs/zh/02-intro.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/02-intro.md b/docs/zh/02-intro.md index 0c5674b761..ea548570d5 100644 --- a/docs/zh/02-intro.md +++ b/docs/zh/02-intro.md @@ -3,7 +3,7 @@ title: 产品简介 toc_max_heading_level: 2 --- -TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/tdengine/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database]的时序数据库 (Time-Series Database, TSDB)。TDengine 能被广泛运用于物联网、工业互联网、车联网、IT 运维、金融等领域。除核心的时序数据库功能外，TDengine 还提供[缓存](/develop/cache/)、[数据订阅](/develop/subscribe)、[流式计算](/develop/continuous-query)等功能，是一极简的时序数据处理平台，最大程度的减小系统设计的复杂度，降低研发和运营成本。 +TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/tdengine/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的时序数据库 (Time-Series Database, TSDB)。TDengine 能被广泛运用于物联网、工业互联网、车联网、IT 运维、金融等领域。除核心的时序数据库功能外，TDengine 还提供[缓存](/develop/cache/)、[数据订阅](/develop/subscribe)、[流式计算](/develop/continuous-query)等功能，是一极简的时序数据处理平台，最大程度的减小系统设计的复杂度，降低研发和运营成本。 本章节介绍TDengine的主要功能、竞争优势、适用场景、与其他数据库的对比测试等等，让大家对TDengine有个整体的了解。