From 31dc5533900b1008e44d342fef6231b953ab2b46 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 9 Jun 2022 13:24:04 +0800 Subject: [PATCH 01/17] feat: sma index optimize --- include/common/tmsg.h | 21 +- include/libs/catalog/catalog.h | 2 +- source/libs/catalog/src/catalog.c | 6 +- source/libs/parser/inc/parUtil.h | 4 +- source/libs/parser/src/parTranslater.c | 319 ++++++++++++++++++- source/libs/parser/src/parUtil.c | 4 + source/libs/parser/test/mockCatalogService.h | 1 + source/libs/planner/inc/planInt.h | 6 +- source/libs/planner/src/planLogicCreater.c | 43 ++- source/libs/planner/src/planOptimizer.c | 4 +- source/libs/planner/src/planSpliter.c | 47 +-- source/libs/planner/src/planner.c | 8 +- source/libs/planner/test/planOtherTest.cpp | 2 +- source/libs/planner/test/planTestUtil.cpp | 40 ++- 14 files changed, 404 insertions(+), 103 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 4a3c4b0c3f..87b94d139c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2493,14 +2493,14 @@ int32_t tSerializeSTableIndexReq(void* buf, int32_t bufLen, STableIndexReq* pReq int32_t tDeserializeSTableIndexReq(void* buf, int32_t bufLen, STableIndexReq* pReq); typedef struct { - int8_t intervalUnit; - int8_t slidingUnit; - int64_t interval; - int64_t offset; - int64_t sliding; - int64_t dstTbUid; - int32_t dstVgId; // for stream - char* expr; + int8_t intervalUnit; + int8_t slidingUnit; + int64_t interval; + int64_t offset; + int64_t sliding; + int64_t dstTbUid; + int32_t dstVgId; + char* expr; } STableIndexInfo; typedef struct { @@ -2510,7 +2510,6 @@ typedef struct { int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp); int32_t tDeserializeSTableIndexRsp(void* buf, int32_t bufLen, STableIndexRsp* pRsp); - typedef struct { int8_t mqMsgType; int32_t code; @@ -2751,8 +2750,8 @@ typedef struct { char* msg; } SVDeleteReq; -int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq); -int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq); +int32_t tSerializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq); +int32_t tDeserializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq); typedef struct { int64_t affectedRows; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 21fee3cc3e..11241c3ae3 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -272,7 +272,7 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); -int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* tbFName, SArray** pRes); +int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pName, SArray** pRes); int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 0f6a79c14c..d9f927abf0 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1136,13 +1136,15 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps CTG_API_LEAVE(ctgGetIndexInfoFromMnode(CTG_PARAMS_LIST(), indexName, pInfo, NULL)); } -int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* tbFName, SArray** pRes) { +int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pName, SArray** pRes) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == tbFName || NULL == pRes) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pName || NULL == pRes) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(pName, tbFName); CTG_API_LEAVE(ctgGetTbIndexFromMnode(CTG_PARAMS_LIST(), tbFName, pRes, NULL)); } diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 0351023f5b..42719d95bd 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -46,6 +46,7 @@ typedef struct SParseMetaCache { SHashObj* pDbInfo; // key is tbFName, element is SDbInfo* SHashObj* pUserAuth; // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass SHashObj* pUdf; // key is funcName, element is SFuncInfo* + SHashObj* pSmaIndex; // key is tbFName, element is SArray* } SParseMetaCache; int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...); @@ -58,7 +59,7 @@ int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); -int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf); +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMsgBuf* pMsgBuf); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); @@ -84,6 +85,7 @@ int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDb int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type, bool* pPass); int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo); +int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f77bbb34f3..9777712134 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -25,6 +25,9 @@ #include "tglobal.h" #include "ttime.h" +#define SMA_TABLE_NAME "#sma_table" +#define SMA_COL_NAME_PREFIX "#sma_col_" + #define generateDealNodeErrMsg(pCxt, code, ...) \ (pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__), DEAL_RES_ERROR) @@ -260,6 +263,28 @@ static int32_t getUdfInfo(STranslateContext* pCxt, SFunctionNode* pFunc) { return code; } +static int32_t getTableIndex(STranslateContext* pCxt, const char* pDbName, const char* pTableName, SArray** pIndexes) { + SParseContext* pParCxt = pCxt->pParseCxt; + SName name; + toName(pParCxt->acctId, pDbName, pTableName, &name); + int32_t code = TSDB_CODE_SUCCESS; + if (pParCxt->async) { + code = getTableIndexFromCache(pCxt->pMetaCache, &name, pIndexes); + } else { + code = collectUseDatabase(&name, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(&name, pCxt->pTables); + } + if (TSDB_CODE_SUCCESS == code) { + code = catalogGetTableIndex(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, &name, pIndexes); + } + } + if (TSDB_CODE_SUCCESS != code) { + parserError("getTableIndex error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pDbName, pTableName); + } + return code; +} + static int32_t initTranslateContext(SParseContext* pParseCxt, SParseMetaCache* pMetaCache, STranslateContext* pCxt) { pCxt->pParseCxt = pParseCxt; pCxt->errCode = TSDB_CODE_SUCCESS; @@ -334,6 +359,10 @@ static bool isIndefiniteRowsFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsIndefiniteRowsFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isVectorFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsVectorFunc(((SFunctionNode*)pNode)->funcId)); +} + static bool isDistinctOrderBy(STranslateContext* pCxt) { return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrSelectStmt->isDistinct); } @@ -1787,7 +1816,7 @@ static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char uni return -1; } -static int32_t checkIntervalWindow(STranslateContext* pCxt, SNode* pWhere, SIntervalWindowNode* pInterval) { +static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; @@ -1829,7 +1858,15 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SNode* pWhere, SInte } } - return translateFill(pCxt, pWhere, pInterval); + return TSDB_CODE_SUCCESS; +} + +static int32_t translateIntervalWindow(STranslateContext* pCxt, SSelectStmt* pSelect, SIntervalWindowNode* pInterval) { + int32_t code = checkIntervalWindow(pCxt, pInterval); + if (TSDB_CODE_SUCCESS == code) { + code = translateFill(pCxt, pSelect->pWhere, pInterval); + } + return code; } static EDealRes checkStateExpr(SNode* pNode, void* pContext) { @@ -1851,13 +1888,13 @@ static EDealRes checkStateExpr(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static int32_t checkStateWindow(STranslateContext* pCxt, SStateWindowNode* pState) { +static int32_t translateStateWindow(STranslateContext* pCxt, SStateWindowNode* pState) { nodesWalkExprPostOrder(pState->pExpr, checkStateExpr, pCxt); // todo check for "function not support for state_window" return pCxt->errCode; } -static int32_t checkSessionWindow(STranslateContext* pCxt, SSessionWindowNode* pSession) { +static int32_t translateSessionWindow(STranslateContext* pCxt, SSessionWindowNode* pSession) { if ('y' == pSession->pGap->unit || 'n' == pSession->pGap->unit || 0 == pSession->pGap->datum.i) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SESSION_GAP); } @@ -1868,14 +1905,14 @@ static int32_t checkSessionWindow(STranslateContext* pCxt, SSessionWindowNode* p return TSDB_CODE_SUCCESS; } -static int32_t checkWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { +static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_STATE_WINDOW: - return checkStateWindow(pCxt, (SStateWindowNode*)pSelect->pWindow); + return translateStateWindow(pCxt, (SStateWindowNode*)pSelect->pWindow); case QUERY_NODE_SESSION_WINDOW: - return checkSessionWindow(pCxt, (SSessionWindowNode*)pSelect->pWindow); + return translateSessionWindow(pCxt, (SSessionWindowNode*)pSelect->pWindow); case QUERY_NODE_INTERVAL_WINDOW: - return checkIntervalWindow(pCxt, pSelect->pWhere, (SIntervalWindowNode*)pSelect->pWindow); + return translateIntervalWindow(pCxt, pSelect, (SIntervalWindowNode*)pSelect->pWindow); default: break; } @@ -1889,7 +1926,7 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_WINDOW; int32_t code = translateExpr(pCxt, &pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { - code = checkWindow(pCxt, pSelect); + code = translateSpecificWindow(pCxt, pSelect); } return code; } @@ -1965,6 +2002,270 @@ static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SSelectStmt* pSelect nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, rewriteTimelineFuncImpl, pCxt); return pCxt->errCode; } +#if 0 +static bool mayBeApplySmaIndex(SSelectStmt* pSelect) { + if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow) || + NULL != ((SIntervalWindowNode*)pSelect->pWindow)->pFill || + QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) || NULL != pSelect->pWhere || + NULL != pSelect->pPartitionByList || NULL != pSelect->pGroupByList || NULL != pSelect->pHaving) { + return false; + } + return true; +} + +static bool equalIntervalWindow(SIntervalWindowNode* pInterval, SNode* pWhere, STableIndexInfo* pIndex) { + int64_t interval = ((SValueNode*)pInterval->pInterval)->datum.i; + int8_t intervalUnit = ((SValueNode*)pInterval->pInterval)->unit; + int64_t offset = (NULL != pInterval->pOffset ? ((SValueNode*)pInterval->pOffset)->datum.i : 0); + int64_t sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : interval); + int8_t slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : intervalUnit); + if (interval != pIndex->interval || intervalUnit != pIndex->intervalUnit || offset != pIndex->offset || + sliding != pIndex->sliding || slidingUnit != pIndex->slidingUnit) { + return false; + } + // todo + if (NULL != pWhere) { + return false; + } + return true; +} + +typedef struct SSmaIndexMatchFuncsCxt { + int32_t errCode; + uint64_t tableId; + SNodeList* pSmaFuncs; + SNodeList* pUseFuncs; + SNodeList* pUseCols; + SArray* pUseMap; + bool match; +} SSmaIndexMatchFuncsCxt; + +static SColumnNode* createColumnFromSmaFunc(uint64_t tableId, int32_t index, SExprNode* pSmaFunc) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return NULL; + } + pCol->tableId = tableId; + pCol->tableType = TSDB_SUPER_TABLE; + pCol->colId = index + 2; // skip timestamp primary col + pCol->colType = COLUMN_TYPE_COLUMN; + snprintf(pCol->colName, sizeof(pCol->colName), SMA_COL_NAME_PREFIX "%d", pCol->colId); + strcpy(pCol->tableName, SMA_TABLE_NAME); + strcpy(pCol->tableAlias, SMA_TABLE_NAME); + pCol->node.resType = pSmaFunc->resType; + strcpy(pCol->node.aliasName, pSmaFunc->aliasName); + return pCol; +} + +static int32_t collectSmaFunc(SSmaIndexMatchFuncsCxt* pCxt, int32_t index, SNode* pSmaFunc) { + if (NULL == pCxt->pUseMap) { + int32_t nfuncs = LIST_LENGTH(pCxt->pSmaFuncs); + pCxt->pUseMap = taosArrayInit(nfuncs, sizeof(int32_t)); + if (NULL == pCxt->pUseMap) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t initPos = -1; + for (int32_t i = 0; i < nfuncs; ++i) { + taosArrayPush(pCxt->pUseMap, &initPos); + } + } + int32_t pos = *(int32_t*)taosArrayGet(pCxt->pUseMap, index); + if (pos < 0) { + pos = LIST_LENGTH(pCxt->pUseFuncs); + taosArraySet(pCxt->pUseMap, index, &pos); + int32_t code = + nodesListMakeStrictAppend(&pCxt->pUseCols, createColumnFromSmaFunc(pCxt->tableId, index, (SExprNode*)pSmaFunc)); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pCxt->pUseFuncs, nodesCloneNode(pSmaFunc)); + } + return code; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t findSmaFunc(SSmaIndexMatchFuncsCxt* pCxt, SNode* pNode, bool* pFound) { + if (!isAggFunc(pNode)) { + return TSDB_CODE_SUCCESS; + } + + int32_t index = 0; + SNode* pSmaFunc = NULL; + FOREACH(pSmaFunc, pCxt->pSmaFuncs) { + if (nodesEqualNode(pSmaFunc, pNode)) { + *pFound = true; + return collectSmaFunc(pCxt, index, pSmaFunc); + } + ++index; + } + return TSDB_CODE_SUCCESS; +} + +static EDealRes matchSmaFuncsImpl(SNode* pNode, void* pContext) { + SSmaIndexMatchFuncsCxt* pCxt = pContext; + + bool found = false; + pCxt->errCode = findSmaFunc(pCxt, pNode, &found); + if (TSDB_CODE_SUCCESS != pCxt->errCode) { + return DEAL_RES_ERROR; + } + + if (found) { + pCxt->match = true; + return DEAL_RES_IGNORE_CHILD; + } + + if (isVectorFunc(pNode)) { + pCxt->match = false; + return DEAL_RES_END; + } + + return DEAL_RES_CONTINUE; +} + +static int32_t matchSmaFuncs(SSelectStmt* pSelect, STableIndexInfo* pIndex, SNodeList* pSmaFuncs, SNodeList** pFuncs, + SNodeList** pCols) { + SSmaIndexMatchFuncsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, + .tableId = pIndex->dstTbUid, + .pSmaFuncs = pSmaFuncs, + .pUseFuncs = NULL, + .pUseCols = NULL, + .pUseMap = NULL, + .match = false}; + nodesWalkExprs(pSelect->pProjectionList, matchSmaFuncsImpl, &cxt); + if (TSDB_CODE_SUCCESS == cxt.errCode && cxt.match) { + *pFuncs = cxt.pUseFuncs; + *pCols = cxt.pUseCols; + } else { + nodesDestroyList(cxt.pUseFuncs); + nodesDestroyList(cxt.pUseCols); + } + taosArrayDestroy(cxt.pUseMap); + return cxt.errCode; +} + +static int32_t couldApplySmaIndex(STranslateContext* pCxt, SSelectStmt* pSelect, STableIndexInfo* pIndex, + SNodeList** pFuncs, SNodeList** pCols) { + if (!equalIntervalWindow((SIntervalWindowNode*)pSelect->pWindow, pSelect->pWhere, pIndex)) { + return TSDB_CODE_SUCCESS; + } + SNodeList* pSmaFuncs = NULL; + int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); + if (TSDB_CODE_SUCCESS == code) { + pCxt->currClause = SQL_CLAUSE_SELECT; + code = translateExprList(pCxt, pSmaFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = matchSmaFuncs(pSelect, pIndex, pSmaFuncs, pFuncs, pCols); + } + nodesDestroyList(pSmaFuncs); + return code; +} + +typedef struct SSmaIndexRewriteFuncsCxt { + int32_t errCode; + SNodeList* pFuncs; + SNodeList* pCols; +} SSmaIndexRewriteFuncsCxt; + +static EDealRes rewriteFuncBySmaIndex(SNode** pNode, void* pContext) { + if (isAggFunc(*pNode)) { + SSmaIndexRewriteFuncsCxt* pCxt = pContext; + SNode* pFunc; + int32_t index = 0; + FOREACH(pFunc, pCxt->pFuncs) { + if (nodesEqualNode(pFunc, *pNode)) { + SNode* pNew = nodesCloneNode(nodesListGetNode(pCxt->pCols, index)); + if (NULL == pNew) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + nodesDestroyNode(*pNode); + *pNode = pNew; + return DEAL_RES_IGNORE_CHILD; + } + ++index; + } + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteTableBySmaIndex(SSelectStmt* pSelect, STableIndexInfo* pMatchIndex) { + nodesDestroyNode(pSelect->pFromTable); + SRealTableNode* pRealTable = nodesMakeNode(QUERY_NODE_REAL_TABLE); + if (NULL == pRealTable) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pRealTable->table.singleTable = true; + strcpy(pRealTable->table.tableName, SMA_TABLE_NAME); + pRealTable->pMeta = taosMemoryCalloc(1, sizeof(STableMeta)); + if (NULL == pRealTable->pMeta) { + nodesDestroyNode(pRealTable); + return TSDB_CODE_OUT_OF_MEMORY; + } + pRealTable->pMeta->vgId = pMatchIndex->dstVgId; + pRealTable->pMeta->uid = pMatchIndex->dstTbUid; + pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo)); + if (NULL == pRealTable->pVgroupList) { + nodesDestroyNode(pRealTable); + return TSDB_CODE_OUT_OF_MEMORY; + } + // todo + pSelect->pFromTable = (SNode*)pRealTable; + return TSDB_CODE_SUCCESS; +} + +static int32_t rewriteSelectBySmaIndex(SSelectStmt* pSelect, STableIndexInfo* pMatchIndex, SNodeList* pFuncs, + SNodeList* pCols) { + SSmaIndexRewriteFuncsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pFuncs = pFuncs, .pCols = pCols}; + nodesRewriteExprs(pSelect->pProjectionList, rewriteFuncBySmaIndex, &cxt); + if (TSDB_CODE_SUCCESS == cxt.errCode) { + cxt.errCode = rewriteTableBySmaIndex(pSelect, pMatchIndex); + } + return cxt.errCode; +} + +static int32_t attemptApplySmaIndexImpl(STranslateContext* pCxt, SSelectStmt* pSelect, SArray* pIndexes) { + if (NULL == pIndexes) { + return TSDB_CODE_SUCCESS; + } + int32_t nindexes = taosArrayGetSize(pIndexes); + for (int32_t i = 0; i < nindexes; ++i) { + STableIndexInfo* pIndex = taosArrayGet(pIndexes, i); + SNodeList* pFuncs = NULL; + SNodeList* pCols = NULL; + if (TSDB_CODE_SUCCESS != couldApplySmaIndex(pCxt, pSelect, pIndex, &pFuncs, &pCols)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (NULL != pFuncs) { + int32_t code = rewriteSelectBySmaIndex(pSelect, pIndex, pFuncs, pCols); + nodesDestroyList(pFuncs); + nodesDestroyList(pCols); + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static void destroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } + +static int32_t attemptApplySmaIndex(STranslateContext* pCxt, SSelectStmt* pSelect) { + SRealTableNode* pRealTable = (SRealTableNode*)pSelect->pFromTable; + SArray* pIndexes = NULL; + int32_t code = getTableIndex(pCxt, pRealTable->table.dbName, pRealTable->table.tableName, &pIndexes); + if (TSDB_CODE_SUCCESS == code) { + code = attemptApplySmaIndexImpl(pCxt, pSelect, pIndexes); + } + taosArrayDestroyEx(pIndexes, destroySmaIndex); + return code; +} + +static int32_t attemptApplyIndex(STranslateContext* pCxt, SSelectStmt* pSelect) { + // if (mayBeApplySmaIndex(pSelect)) { + // return attemptApplySmaIndex(pCxt, pSelect); + // } + return TSDB_CODE_SUCCESS; +} +#endif static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrSelectStmt = pSelect; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 147ff933fb..b4564e967d 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -836,3 +836,7 @@ int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFun memcpy(pInfo, *pRes, sizeof(SFuncInfo)); return TSDB_CODE_SUCCESS; } + +int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes) { + return TSDB_CODE_PAR_INTERNAL_ERROR; +} diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index 133a355c59..fafad81e95 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -57,6 +57,7 @@ class MockCatalogService { void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid); void showTables() const; void createFunction(const std::string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize); + void createSmaIndex(const SMCreateSmaReq* pReq); int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const; int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const; diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 1a8c7657df..6cfef2a7fb 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -38,9 +38,9 @@ extern "C" { int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...); int32_t createColumnByRewriteExps(SNodeList* pExprs, SNodeList** pList); -int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode); -int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode); -int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan); +int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan); +int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan); +int32_t splitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan); int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan); int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index a9f3909af6..5ae4687686 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1138,11 +1138,40 @@ static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogi return TSDB_CODE_FAILED; } -int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) { - SLogicPlanContext cxt = {.pPlanCxt = pCxt}; - int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, pLogicNode); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - return TSDB_CODE_SUCCESS; +static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { + pNode->pParent = pParent; + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { doSetLogicNodeParent((SLogicNode*)pChild, pNode); } +} + +static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); } + +int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { + SLogicPlanContext cxt = {.pPlanCxt = pCxt}; + + SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); + if (NULL == pSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pSubplan->id.queryId = pCxt->queryId; + pSubplan->id.groupId = 1; + pSubplan->id.subplanId = 1; + + int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, &pSubplan->pNode); + if (TSDB_CODE_SUCCESS == code) { + setLogicNodeParent(pSubplan->pNode); + if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pSubplan->pNode)) { + pSubplan->subplanType = SUBPLAN_TYPE_MODIFY; + } else { + pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicSubplan = pSubplan; + } else { + nodesDestroyNode(pSubplan); + } + + return code; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8346aca76f..f252fe7265 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -773,4 +773,6 @@ static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicNode* pLogicNode) { return TSDB_CODE_SUCCESS; } -int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode) { return applyOptimizeRule(pCxt, pLogicNode); } +int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { + return applyOptimizeRule(pCxt, pLogicSubplan->pNode); +} diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index e5c81eda4b..97bb9389f2 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -915,14 +915,6 @@ static int32_t applySplitRule(SPlanContext* pCxt, SLogicSubplan* pSubplan) { return TSDB_CODE_SUCCESS; } -static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { - pNode->pParent = pParent; - SNode* pChild; - FOREACH(pChild, pNode->pChildren) { doSetLogicNodeParent((SLogicNode*)pChild, pNode); } -} - -static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); } - static void setVgroupsInfo(SLogicNode* pNode, SLogicSubplan* pSubplan) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { TSWAP(((SScanLogicNode*)pNode)->pVgroupList, pSubplan->pVgroupList); @@ -933,37 +925,10 @@ static void setVgroupsInfo(SLogicNode* pNode, SLogicSubplan* pSubplan) { FOREACH(pChild, pNode->pChildren) { setVgroupsInfo((SLogicNode*)pChild, pSubplan); } } -int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan) { - SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); - if (NULL == pSubplan) { - return TSDB_CODE_OUT_OF_MEMORY; +int32_t splitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { + if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pLogicSubplan->pNode)) { + setVgroupsInfo(pLogicSubplan->pNode, pLogicSubplan); + return TSDB_CODE_SUCCESS; } - - pSubplan->pNode = nodesCloneNode(pLogicNode); - if (NULL == pSubplan->pNode) { - nodesDestroyNode(pSubplan); - return TSDB_CODE_OUT_OF_MEMORY; - } - - pSubplan->id.queryId = pCxt->queryId; - pSubplan->id.groupId = 1; - setLogicNodeParent(pSubplan->pNode); - - int32_t code = TSDB_CODE_SUCCESS; - if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pLogicNode)) { - pSubplan->subplanType = SUBPLAN_TYPE_MODIFY; - TSWAP(((SVnodeModifyLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifyLogicNode*)pSubplan->pNode)->pDataBlocks); - setVgroupsInfo(pSubplan->pNode, pSubplan); - } else { - pSubplan->subplanType = SUBPLAN_TYPE_SCAN; - code = applySplitRule(pCxt, pSubplan); - } - - if (TSDB_CODE_SUCCESS == code) { - *pLogicSubplan = pSubplan; - } else { - nodesDestroyNode(pSubplan); - } - - return code; -} \ No newline at end of file + return applySplitRule(pCxt, pLogicSubplan); +} diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 1921b16388..83657d27d0 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -26,16 +26,15 @@ static void dumpQueryPlan(SQueryPlan* pPlan) { } int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList) { - SLogicNode* pLogicNode = NULL; SLogicSubplan* pLogicSubplan = NULL; SQueryLogicPlan* pLogicPlan = NULL; - int32_t code = createLogicPlan(pCxt, &pLogicNode); + int32_t code = createLogicPlan(pCxt, &pLogicSubplan); if (TSDB_CODE_SUCCESS == code) { - code = optimizeLogicPlan(pCxt, pLogicNode); + code = optimizeLogicPlan(pCxt, pLogicSubplan); } if (TSDB_CODE_SUCCESS == code) { - code = splitLogicPlan(pCxt, pLogicNode, &pLogicSubplan); + code = splitLogicPlan(pCxt, pLogicSubplan); } if (TSDB_CODE_SUCCESS == code) { code = scaleOutLogicPlan(pCxt, pLogicSubplan, &pLogicPlan); @@ -47,7 +46,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo dumpQueryPlan(*pPlan); } - nodesDestroyNode(pLogicNode); nodesDestroyNode(pLogicSubplan); nodesDestroyNode(pLogicPlan); terrno = code; diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index d5d37fda64..04d76a8b91 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -42,7 +42,7 @@ TEST_F(PlanOtherTest, createStreamUseSTable) { TEST_F(PlanOtherTest, createSmaIndex) { useDb("root", "test"); - run("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) interval(10s)"); + run("CREATE SMA INDEX idx1 ON t1 FUNCTION(MAX(c1), MIN(c3 + 10), SUM(c4)) INTERVAL(10s)"); } TEST_F(PlanOtherTest, explain) { diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index f5c8b58e43..4e89f9d008 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -104,13 +104,12 @@ class PlannerTestBaseImpl { SPlanContext cxt = {0}; setPlanContext(pQuery, &cxt); - SLogicNode* pLogicNode = nullptr; - doCreateLogicPlan(&cxt, &pLogicNode); - - doOptimizeLogicPlan(&cxt, pLogicNode); - SLogicSubplan* pLogicSubplan = nullptr; - doSplitLogicPlan(&cxt, pLogicNode, &pLogicSubplan); + doCreateLogicPlan(&cxt, &pLogicSubplan); + + doOptimizeLogicPlan(&cxt, pLogicSubplan); + + doSplitLogicPlan(&cxt, pLogicSubplan); SQueryLogicPlan* pLogicPlan = nullptr; doScaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan); @@ -164,13 +163,12 @@ class PlannerTestBaseImpl { SPlanContext cxt = {0}; setPlanContext(stmtEnv_.pQuery_, &cxt); - SLogicNode* pLogicNode = nullptr; - doCreateLogicPlan(&cxt, &pLogicNode); - - doOptimizeLogicPlan(&cxt, pLogicNode); - SLogicSubplan* pLogicSubplan = nullptr; - doSplitLogicPlan(&cxt, pLogicNode, &pLogicSubplan); + doCreateLogicPlan(&cxt, &pLogicSubplan); + + doOptimizeLogicPlan(&cxt, pLogicSubplan); + + doSplitLogicPlan(&cxt, pLogicSubplan); SQueryLogicPlan* pLogicPlan = nullptr; doScaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan); @@ -324,19 +322,19 @@ class PlannerTestBaseImpl { res_.ast_ = toString(pQuery->pRoot); } - void doCreateLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) { - DO_WITH_THROW(createLogicPlan, pCxt, pLogicNode); - res_.rawLogicPlan_ = toString((SNode*)(*pLogicNode)); + void doCreateLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { + DO_WITH_THROW(createLogicPlan, pCxt, pLogicSubplan); + res_.rawLogicPlan_ = toString((SNode*)(*pLogicSubplan)); } - void doOptimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode) { - DO_WITH_THROW(optimizeLogicPlan, pCxt, pLogicNode); - res_.optimizedLogicPlan_ = toString((SNode*)pLogicNode); + void doOptimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { + DO_WITH_THROW(optimizeLogicPlan, pCxt, pLogicSubplan); + res_.optimizedLogicPlan_ = toString((SNode*)pLogicSubplan); } - void doSplitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan) { - DO_WITH_THROW(splitLogicPlan, pCxt, pLogicNode, pLogicSubplan); - res_.splitLogicPlan_ = toString((SNode*)(*pLogicSubplan)); + void doSplitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { + DO_WITH_THROW(splitLogicPlan, pCxt, pLogicSubplan); + res_.splitLogicPlan_ = toString((SNode*)(pLogicSubplan)); } void doScaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan) { From 25636d6201e929b9657f51f7d2f7cbac4e79239b Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 9 Jun 2022 19:21:52 +0800 Subject: [PATCH 02/17] feature: add merge interval operator --- source/libs/executor/inc/executorimpl.h | 3 + source/libs/executor/src/executorimpl.c | 45 ++++ source/libs/executor/src/timewindowoperator.c | 226 ++++++++++++++++++ 3 files changed, 274 insertions(+) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 4f02c559b1..5dd349f4ab 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -895,6 +895,9 @@ int64_t getSmaWaterMark(int64_t interval, double filesFactor); bool isSmaStream(int8_t triggerType); int32_t compareTimeWindow(const void* p1, const void* p2, const void* param); +int32_t finalizeResultRowIntoSDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, + SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, + SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7b73fd8ae9..5bd9044167 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1955,6 +1955,51 @@ static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_ } } +int32_t finalizeResultRowIntoSDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, + SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, + SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); + SResultRow* pRow = (SResultRow*)((char*)page + resultRowPosition->offset); + + doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset); + if (pRow->numOfRows == 0) { + releaseBufPage(pBuf, page); + return 0; + } + + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + releaseBufPage(pBuf, page); + return -1; + } + + for (int32_t j = 0; j < numOfExprs; ++j) { + int32_t slotId = pExprInfo[j].base.resSchema.slotId; + + pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset); + if (pCtx[j].fpSet.finalize) { + int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (TAOS_FAILED(code)) { + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); + longjmp(pTaskInfo->env, code); + } + } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { + // do nothing, todo refactor + } else { + // expand the result into multiple rows. E.g., _wstartts, top(k, 20) + // the _wstartts needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } + } + } + + releaseBufPage(pBuf, page); + + return 0; +} + int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b309478556..696b9139cf 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3175,3 +3175,229 @@ _error: pTaskInfo->code = code; return NULL; } + +typedef struct SMergeIntervalAggOperatorInfo { + SIntervalAggOperatorInfo intervalAggOperatorInfo; + + SHashObj* groupIntervalHash; +} SMergeIntervalAggOperatorInfo; + +void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { + SMergeIntervalAggOperatorInfo* pInfo = (SMergeIntervalAggOperatorInfo*)param; + taosHashCleanup(pInfo->groupIntervalHash); + destroyIntervalOperatorInfo(&pInfo->intervalAggOperatorInfo, numOfOutput); +} + +static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, + int32_t scanFlag, SSDataBlock* pResultBlock) { + SMergeIntervalAggOperatorInfo *miaInfo = pOperatorInfo->info; + SIntervalAggOperatorInfo * pInfo = &miaInfo->intervalAggOperatorInfo; + + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + + int32_t startPos = 0; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + int64_t* tsCols = extractTsCol(pBlock, pInfo); + uint64_t tableGroupId = pBlock->info.groupId; + bool ascScan = (pInfo->order == TSDB_ORDER_ASC); + TSKEY blockStartTs = getStartTsKey(&pBlock->info.window, tsCols); + SResultRow* pResult = NULL; + + STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &pInfo->interval, + pInfo->interval.precision, &pInfo->win); + //TODO: pResultBlock full + //TODO: pBlock not process not finished + //TODO: different block group id or no group id + //TODO: lastWin may be none, p1 shall not be null + //TODO: the last datablock + //TODO: blockDataUpdateTsWindow(pBlock, 0); + + int32_t ret = + setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + TSKEY ekey = ascScan ? win.ekey : win.skey; + int32_t forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); + ASSERT(forwardRows > 0); + + // prev time window not interpolation yet. + if (pInfo->timeWindowInterpo) { + SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult); + doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos); + + // restore current time window + ret = + setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + // window start key interpolation + doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &win, startPos, forwardRows); + } + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, + pBlock->info.rows, numOfOutput, pInfo->order); + + doCloseWindow(pResultRowInfo, pInfo, pResult); + STimeWindow *lastWin = taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); + if (ascScan && win.skey > lastWin->ekey || (!ascScan) && win.skey < lastWin->ekey) { + SET_RES_WINDOW_KEY(pInfo->aggSup.keyBuf, &lastWin->skey, TSDB_KEYSIZE, tableGroupId); + SResultRowPosition* p1 = + (SResultRowPosition*)taosHashGet(pInfo->aggSup.pResultRowHashTable, pInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + finalizeResultRowIntoSDataBlock(pInfo->aggSup.pResultBuf, p1, + pInfo->binfo.pCtx, pOperatorInfo->pExpr, pOperatorInfo->numOfExprs, pInfo->binfo.rowCellInfoOffset, + pResultBlock, pTaskInfo); + taosHashRemove(pInfo->aggSup.pResultRowHashTable, pInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + + taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), &win, sizeof(STimeWindow)); + } + + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = forwardRows - 1 + startPos; + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->order); + if (startPos < 0) { + break; + } + + // null data, failed to allocate more memory buffer + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, + &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ekey = ascScan ? nextWin.ekey : nextWin.skey; + forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); + + // window start(end) key interpolation + doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, + forwardRows); + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + tsCols, pBlock->info.rows, numOfOutput, pInfo->order); + doCloseWindow(pResultRowInfo, pInfo, pResult); + } + + if (pInfo->timeWindowInterpo) { + saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols); + } +} + +static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info; + SIntervalAggOperatorInfo *pInfo = &miaInfo->intervalAggOperatorInfo; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSDataBlock* pRes = pInfo->binfo.pRes; + blockDataCleanup(pRes); + + int32_t scanFlag = MAIN_SCAN; + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + break; + } + + getTableScanInfo(pOperator, &pInfo->order, &scanFlag); + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, scanFlag, true); + STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; + + setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); + doMergeIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); + } + + if (pRes->info.rows == 0) { + doSetOperatorCompleted(pOperator); + } + + size_t rows = pRes->info.rows; + pOperator->resultInfo.totalRows += rows; + return (rows == 0) ? NULL : pRes; +} + +SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { + SMergeIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (miaInfo == NULL || pOperator == NULL) { + goto _error; + } + SIntervalAggOperatorInfo *pInfo = &miaInfo->intervalAggOperatorInfo; + + pInfo->win = pTaskInfo->window; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + pInfo->execModel = pTaskInfo->execModel; + pInfo->twAggSup = *pTwAggSupp; + + pInfo->primaryTsIndex = primaryTsSlotId; + miaInfo->groupIntervalHash = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(pOperator, 4096); + + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + + pInfo->timeWindowInterpo = timeWindowinterpNeeded(pInfo->binfo.pCtx, numOfCols, pInfo); + if (pInfo->timeWindowInterpo) { + pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition)); + } + + // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "TimeMergeIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, doStreamIntervalAgg, NULL, + destroyIntervalOperatorInfo, NULL, NULL, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + +_error: + destroyMergeIntervalOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} From c97dcbcf76e5d0f2b1520f664026526e0cc91eed Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 10 Jun 2022 09:55:14 +0800 Subject: [PATCH 03/17] feature: add merge interval operator --- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 12 +- source/libs/executor/src/timewindowoperator.c | 155 +++++++++++------- 3 files changed, 108 insertions(+), 61 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 5dd349f4ab..6a7e71412e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -895,7 +895,7 @@ int64_t getSmaWaterMark(int64_t interval, double filesFactor); bool isSmaStream(int8_t triggerType); int32_t compareTimeWindow(const void* p1, const void* p2, const void* param); -int32_t finalizeResultRowIntoSDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, +int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5bd9044167..e82c94073c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1955,7 +1955,7 @@ static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_ } } -int32_t finalizeResultRowIntoSDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, +int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); @@ -1967,9 +1967,13 @@ int32_t finalizeResultRowIntoSDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* return 0; } - if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - releaseBufPage(pBuf, page); - return -1; + while (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + int32_t code = blockDataEnsureCapacity(pBlock, pBlock->info.capacity * 1.25); + if (TAOS_FAILED(code)) { + releaseBufPage(pBuf, page); + qError("%s ensure result data capacity failed, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); + longjmp(pTaskInfo->env, code); + } } for (int32_t j = 0; j < numOfExprs; ++j) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 696b9139cf..3ed416ec7c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3180,117 +3180,140 @@ typedef struct SMergeIntervalAggOperatorInfo { SIntervalAggOperatorInfo intervalAggOperatorInfo; SHashObj* groupIntervalHash; + bool hasGroupId; + uint64_t groupId; + SSDataBlock *prefetchedBlock; } SMergeIntervalAggOperatorInfo; void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { - SMergeIntervalAggOperatorInfo* pInfo = (SMergeIntervalAggOperatorInfo*)param; - taosHashCleanup(pInfo->groupIntervalHash); - destroyIntervalOperatorInfo(&pInfo->intervalAggOperatorInfo, numOfOutput); + SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param; + taosHashCleanup(miaInfo->groupIntervalHash); + destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); +} + +static int32_t outputPrevIntervalResult(SOperatorInfo * pOperatorInfo, uint64_t tableGroupId, SSDataBlock *pResultBlock, STimeWindow* newWin) { + SMergeIntervalAggOperatorInfo *miaInfo = pOperatorInfo->info; + SIntervalAggOperatorInfo * iaInfo = &miaInfo->intervalAggOperatorInfo; + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); + + STimeWindow *prevWin= taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); + if (prevWin == NULL) { + taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); + return 0; + } + + if (ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey) { + SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &prevWin->skey, TSDB_KEYSIZE, tableGroupId); + SResultRowPosition* p1 = + (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + ASSERT(p1 != NULL); + + finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, iaInfo->binfo.pCtx, pOperatorInfo->pExpr, + pOperatorInfo->numOfExprs, iaInfo->binfo.rowCellInfoOffset, pResultBlock, + pTaskInfo); + taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + + taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); + } + return 0; } static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, int32_t scanFlag, SSDataBlock* pResultBlock) { SMergeIntervalAggOperatorInfo *miaInfo = pOperatorInfo->info; - SIntervalAggOperatorInfo * pInfo = &miaInfo->intervalAggOperatorInfo; + SIntervalAggOperatorInfo * iaInfo = &miaInfo->intervalAggOperatorInfo; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; int32_t startPos = 0; int32_t numOfOutput = pOperatorInfo->numOfExprs; - int64_t* tsCols = extractTsCol(pBlock, pInfo); + int64_t* tsCols = extractTsCol(pBlock, iaInfo); uint64_t tableGroupId = pBlock->info.groupId; - bool ascScan = (pInfo->order == TSDB_ORDER_ASC); + bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); TSKEY blockStartTs = getStartTsKey(&pBlock->info.window, tsCols); SResultRow* pResult = NULL; - STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &pInfo->interval, - pInfo->interval.precision, &pInfo->win); - //TODO: pResultBlock full + STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval, + iaInfo->interval.precision, &iaInfo->win); //TODO: pBlock not process not finished //TODO: different block group id or no group id - //TODO: lastWin may be none, p1 shall not be null //TODO: the last datablock //TODO: blockDataUpdateTsWindow(pBlock, 0); int32_t ret = - setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, iaInfo->binfo.pCtx, + numOfOutput, iaInfo->binfo.rowCellInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } TSKEY ekey = ascScan ? win.ekey : win.skey; int32_t forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->order); ASSERT(forwardRows > 0); // prev time window not interpolation yet. - if (pInfo->timeWindowInterpo) { + if (iaInfo->timeWindowInterpo) { SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult); doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos); // restore current time window ret = - setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + iaInfo->binfo.pCtx, + numOfOutput, iaInfo->binfo.rowCellInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } // window start key interpolation - doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &win, startPos, forwardRows); + doWindowBorderInterpolation(iaInfo, pBlock, numOfOutput, iaInfo->binfo.pCtx, pResult, &win, startPos, forwardRows); } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, - pBlock->info.rows, numOfOutput, pInfo->order); + updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pTaskInfo, iaInfo->binfo.pCtx, &win, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, + pBlock->info.rows, numOfOutput, iaInfo->order); + doCloseWindow(pResultRowInfo, iaInfo, pResult); - doCloseWindow(pResultRowInfo, pInfo, pResult); - STimeWindow *lastWin = taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); - if (ascScan && win.skey > lastWin->ekey || (!ascScan) && win.skey < lastWin->ekey) { - SET_RES_WINDOW_KEY(pInfo->aggSup.keyBuf, &lastWin->skey, TSDB_KEYSIZE, tableGroupId); - SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pInfo->aggSup.pResultRowHashTable, pInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); - finalizeResultRowIntoSDataBlock(pInfo->aggSup.pResultBuf, p1, - pInfo->binfo.pCtx, pOperatorInfo->pExpr, pOperatorInfo->numOfExprs, pInfo->binfo.rowCellInfoOffset, - pResultBlock, pTaskInfo); - taosHashRemove(pInfo->aggSup.pResultRowHashTable, pInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); - - taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), &win, sizeof(STimeWindow)); - } + // output previous interval results after this interval (&win) is closed + outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win); STimeWindow nextWin = win; while (1) { int32_t prevEndPos = forwardRows - 1 + startPos; - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->order); + startPos = getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->order); if (startPos < 0) { break; } // null data, failed to allocate more memory buffer int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, - &pInfo->aggSup, pTaskInfo); + iaInfo->binfo.pCtx, numOfOutput, iaInfo->binfo.rowCellInfoOffset, + &iaInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } ekey = ascScan ? nextWin.ekey : nextWin.skey; forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->order); // window start(end) key interpolation - doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, + doWindowBorderInterpolation(iaInfo, pBlock, numOfOutput, iaInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardRows); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, - tsCols, pBlock->info.rows, numOfOutput, pInfo->order); - doCloseWindow(pResultRowInfo, pInfo, pResult); + updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, iaInfo->binfo.pCtx, &nextWin, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, + tsCols, pBlock->info.rows, numOfOutput, iaInfo->order); + doCloseWindow(pResultRowInfo, iaInfo, pResult); + + // output previous interval results after this interval (&nextWin) is closed + outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin); } - if (pInfo->timeWindowInterpo) { - saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols); + if (iaInfo->timeWindowInterpo) { + saveDataBlockLastRow(iaInfo->pPrevValues, pBlock, iaInfo->pInterpCols); } } @@ -3299,36 +3322,56 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info; - SIntervalAggOperatorInfo *pInfo = &miaInfo->intervalAggOperatorInfo; + SIntervalAggOperatorInfo *iaInfo = &miaInfo->intervalAggOperatorInfo; if (pOperator->status == OP_EXEC_DONE) { return NULL; } - SSDataBlock* pRes = pInfo->binfo.pRes; + SSDataBlock* pRes = iaInfo->binfo.pRes; blockDataCleanup(pRes); - int32_t scanFlag = MAIN_SCAN; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - + int32_t scanFlag = MAIN_SCAN; while (1) { - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + SSDataBlock *pBlock = NULL; + if (miaInfo->prefetchedBlock == NULL) { + pBlock = downstream->fpSet.getNextFn(downstream); + } else { + pBlock = miaInfo->prefetchedBlock; + miaInfo->groupId = pBlock->info.groupId; + } + if (pBlock == NULL) { break; } - getTableScanInfo(pOperator, &pInfo->order, &scanFlag); - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, scanFlag, true); - STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; + if (!miaInfo->hasGroupId) { + miaInfo->hasGroupId = true; + miaInfo->groupId = pBlock->info.groupId; + } else if (miaInfo->groupId != pBlock->info.groupId) { + miaInfo->prefetchedBlock = pBlock; + break; + } + + getTableScanInfo(pOperator, &iaInfo->order, &scanFlag); + setInputDataBlock(pOperator, iaInfo->binfo.pCtx, pBlock, iaInfo->order, scanFlag, true); + STableQueryInfo* pTableQueryInfo = iaInfo->pCurrent; setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - doMergeIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); + doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); + + if (pRes->info.rows >= pOperator->resultInfo.threshold) { + break; + } } + pRes->info.groupId = miaInfo->groupId; if (pRes->info.rows == 0) { doSetOperatorCompleted(pOperator); + } else { + //TODO: ts column index + blockDataUpdateTsWindow(pRes, 0); } size_t rows = pRes->info.rows; @@ -3384,7 +3427,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI pOperator->numOfExprs = numOfCols; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, doStreamIntervalAgg, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL, destroyIntervalOperatorInfo, NULL, NULL, NULL); code = appendDownstream(pOperator, &downstream, 1); From 317cc8fc13fcbd6e82ab4714e883f21bfff30840 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 10 Jun 2022 10:16:18 +0800 Subject: [PATCH 04/17] feat: sma index optimize --- include/libs/nodes/plannodes.h | 1 + include/libs/nodes/querynodes.h | 1 + include/util/tjson.h | 15 +- source/libs/nodes/src/nodesCodeFuncs.c | 82 ++++++++++ source/libs/parser/inc/parUtil.h | 3 +- source/libs/parser/src/parAstParser.c | 4 + source/libs/parser/src/parTranslater.c | 82 +++------- source/libs/parser/src/parUtil.c | 40 ++++- source/libs/parser/test/mockCatalog.cpp | 6 + .../libs/parser/test/mockCatalogService.cpp | 59 ++++++- source/libs/parser/test/mockCatalogService.h | 1 + source/libs/planner/src/planOptimizer.c | 144 ++++++++++++++++-- source/libs/planner/test/planOtherTest.cpp | 2 + source/libs/planner/test/planTestUtil.cpp | 3 + source/util/src/tjson.c | 53 ++++++- 15 files changed, 394 insertions(+), 102 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 00380724b3..b62854db43 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -62,6 +62,7 @@ typedef struct SScanLogicNode { int64_t watermark; int16_t tsColId; double filesFactor; + SArray* pSmaIndexes; } SScanLogicNode; typedef struct SJoinLogicNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index c2aa86e89f..33cc4a9c86 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -144,6 +144,7 @@ typedef struct SRealTableNode { SVgroupsInfo* pVgroupList; char qualDbName[TSDB_DB_NAME_LEN]; // SHOW qualDbName.TABLES double ratio; + SArray* pSmaIndexes; } SRealTableNode; typedef struct STempTableNode { diff --git a/include/util/tjson.h b/include/util/tjson.h index 84f7b81726..df433227ca 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -17,16 +17,17 @@ #define _TD_UTIL_JSON_H_ #include "os.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { #endif -#define tjsonGetNumberValue(pJson, pName, val, code) \ - do { \ - uint64_t _tmp = 0; \ +#define tjsonGetNumberValue(pJson, pName, val, code) \ + do { \ + uint64_t _tmp = 0; \ code = tjsonGetBigIntValue(pJson, pName, &_tmp); \ - val = _tmp; \ + val = _tmp; \ } while (0) typedef void SJson; @@ -66,18 +67,20 @@ typedef int32_t (*FToJson)(const void* pObj, SJson* pJson); int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj); int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj); int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* pArray, int32_t itemSize, int32_t num); +int32_t tjsonAddTArray(SJson* pJson, const char* pName, FToJson func, const SArray* pArray); typedef int32_t (*FToObject)(const SJson* pJson, void* pObj); int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj); int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize); int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize); +int32_t tjsonToTArray(const SJson* pJson, const char* pName, FToObject func, SArray** pArray, int32_t itemSize); char* tjsonToString(const SJson* pJson); char* tjsonToUnformattedString(const SJson* pJson); -SJson* tjsonParse(const char* pStr); -bool tjsonValidateJson(const char* pJson); +SJson* tjsonParse(const char* pStr); +bool tjsonValidateJson(const char* pJson); const char* tjsonGetError(); #ifdef __cplusplus diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0735092c1a..48840c4a8b 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2809,10 +2809,85 @@ static int32_t jsonToTableNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkTableIndexInfoIntervalUnit = "IntervalUnit"; +static const char* jkTableIndexInfoSlidingUnit = "SlidingUnit"; +static const char* jkTableIndexInfoInterval = "Interval"; +static const char* jkTableIndexInfoOffset = "Offset"; +static const char* jkTableIndexInfoSliding = "Sliding"; +static const char* jkTableIndexInfoDstTbUid = "DstTbUid"; +static const char* jkTableIndexInfoDstVgId = "DstVgId"; +static const char* jkTableIndexInfoEpSet = "EpSet"; +static const char* jkTableIndexInfoExpr = "Expr"; + +static int32_t tableIndexInfoToJson(const void* pObj, SJson* pJson) { + const STableIndexInfo* pNode = (const STableIndexInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoIntervalUnit, pNode->intervalUnit); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoSlidingUnit, pNode->slidingUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoInterval, pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoOffset, pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoSliding, pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoDstTbUid, pNode->dstTbUid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoDstVgId, pNode->dstVgId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkTableIndexInfoEpSet, epSetToJson, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkTableIndexInfoExpr, pNode->expr); + } + + return code; +} + +static int32_t jsonToTableIndexInfo(const SJson* pJson, void* pObj) { + STableIndexInfo* pNode = (STableIndexInfo*)pObj; + + int32_t code = tjsonGetTinyIntValue(pJson, jkTableIndexInfoIntervalUnit, &pNode->intervalUnit); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkTableIndexInfoSlidingUnit, &pNode->slidingUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoInterval, &pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoOffset, &pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoSliding, &pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoDstTbUid, &pNode->dstTbUid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkTableIndexInfoDstVgId, &pNode->dstVgId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkTableIndexInfoEpSet, jsonToEpSet, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonDupStringValue(pJson, jkTableIndexInfoExpr, &pNode->expr); + } + + return code; +} + static const char* jkRealTableMetaSize = "MetaSize"; static const char* jkRealTableMeta = "Meta"; static const char* jkRealTableVgroupsInfoSize = "VgroupsInfoSize"; static const char* jkRealTableVgroupsInfo = "VgroupsInfo"; +static const char* jkRealTableSmaIndexes = "SmaIndexes"; static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) { const SRealTableNode* pNode = (const SRealTableNode*)pObj; @@ -2830,6 +2905,9 @@ static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkRealTableVgroupsInfo, vgroupsInfoToJson, pNode->pVgroupList); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddTArray(pJson, jkRealTableSmaIndexes, tableIndexInfoToJson, pNode->pSmaIndexes); + } return code; } @@ -2851,6 +2929,10 @@ static int32_t jsonToRealTableNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonMakeObject(pJson, jkRealTableVgroupsInfo, jsonToVgroupsInfo, (void**)&pNode->pVgroupList, objSize); } + if (TSDB_CODE_SUCCESS == code) { + code = + tjsonToTArray(pJson, jkRealTableSmaIndexes, jsonToTableIndexInfo, &pNode->pSmaIndexes, sizeof(STableIndexInfo)); + } return code; } diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 42719d95bd..fb67a35368 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -46,7 +46,7 @@ typedef struct SParseMetaCache { SHashObj* pDbInfo; // key is tbFName, element is SDbInfo* SHashObj* pUserAuth; // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass SHashObj* pUdf; // key is funcName, element is SFuncInfo* - SHashObj* pSmaIndex; // key is tbFName, element is SArray* + SHashObj* pTableIndex; // key is tbFName, element is SArray* } SParseMetaCache; int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...); @@ -76,6 +76,7 @@ int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pD SParseMetaCache* pMetaCache); int32_t reserveUserAuthInCacheExt(const char* pUser, const SName* pName, AUTH_TYPE type, SParseMetaCache* pMetaCache); int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache); +int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta); int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo); int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup); diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 6555ec3a7d..6acc4575e7 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -129,6 +129,10 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, SRealTa if (TSDB_CODE_SUCCESS == code) { code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code) { + code = reserveTableIndexInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, + pCxt->pMetaCache); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 90da79266f..c2fa1ffd22 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -258,24 +258,19 @@ static int32_t getUdfInfo(STranslateContext* pCxt, SFunctionNode* pFunc) { return code; } -static int32_t getTableIndex(STranslateContext* pCxt, const char* pDbName, const char* pTableName, SArray** pIndexes) { +static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray** pIndexes) { SParseContext* pParCxt = pCxt->pParseCxt; - SName name; - toName(pParCxt->acctId, pDbName, pTableName, &name); - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = collectUseDatabase(pName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(pName, pCxt->pTables); + } if (pParCxt->async) { - code = getTableIndexFromCache(pCxt->pMetaCache, &name, pIndexes); + code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes); } else { - code = collectUseDatabase(&name, pCxt->pDbs); - if (TSDB_CODE_SUCCESS == code) { - code = collectUseTable(&name, pCxt->pTables); - } - if (TSDB_CODE_SUCCESS == code) { - code = catalogGetTableIndex(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, &name, pIndexes); - } + code = catalogGetTableIndex(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pIndexes); } if (TSDB_CODE_SUCCESS != code) { - parserError("getTableIndex error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pDbName, pTableName); + parserError("getTableIndex error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pName->dbname, pName->tname); } return code; } @@ -853,9 +848,9 @@ static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNo } if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { SNodeListNode* pRight = (SNodeListNode*)pOp->pRight; - bool first = true; - SDataType targetDt = {0}; - SNode* pNode = NULL; + bool first = true; + SDataType targetDt = {0}; + SNode* pNode = NULL; FOREACH(pNode, pRight->pNodeList) { SDataType dt = ((SExprNode*)pNode)->resType; if (first) { @@ -1409,6 +1404,9 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pRealTable->table.tableName); } code = setTableVgroupList(pCxt, &name, pRealTable); + if (TSDB_CODE_SUCCESS == code) { + code = getTableIndex(pCxt, &name, &pRealTable->pSmaIndexes); + } } pRealTable->table.precision = pRealTable->pMeta->tableInfo.precision; pRealTable->table.singleTable = isSingleTable(pRealTable); @@ -2018,33 +2016,8 @@ static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SSelectStmt* pSelect nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, rewriteTimelineFuncImpl, pCxt); return pCxt->errCode; } -#if 0 -static bool mayBeApplySmaIndex(SSelectStmt* pSelect) { - if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow) || - NULL != ((SIntervalWindowNode*)pSelect->pWindow)->pFill || - QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) || NULL != pSelect->pWhere || - NULL != pSelect->pPartitionByList || NULL != pSelect->pGroupByList || NULL != pSelect->pHaving) { - return false; - } - return true; -} -static bool equalIntervalWindow(SIntervalWindowNode* pInterval, SNode* pWhere, STableIndexInfo* pIndex) { - int64_t interval = ((SValueNode*)pInterval->pInterval)->datum.i; - int8_t intervalUnit = ((SValueNode*)pInterval->pInterval)->unit; - int64_t offset = (NULL != pInterval->pOffset ? ((SValueNode*)pInterval->pOffset)->datum.i : 0); - int64_t sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : interval); - int8_t slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : intervalUnit); - if (interval != pIndex->interval || intervalUnit != pIndex->intervalUnit || offset != pIndex->offset || - sliding != pIndex->sliding || slidingUnit != pIndex->slidingUnit) { - return false; - } - // todo - if (NULL != pWhere) { - return false; - } - return true; -} +#if 0 typedef struct SSmaIndexMatchFuncsCxt { int32_t errCode; @@ -2056,10 +2029,10 @@ typedef struct SSmaIndexMatchFuncsCxt { bool match; } SSmaIndexMatchFuncsCxt; -static SColumnNode* createColumnFromSmaFunc(uint64_t tableId, int32_t index, SExprNode* pSmaFunc) { +static int32_t smaOptCreateSmaCol(SNode* pSmaFunc, int32_t index, SNode** pOutput) { SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { - return NULL; + return TSDB_CODE_SUCCESS; } pCol->tableId = tableId; pCol->tableType = TSDB_SUPER_TABLE; @@ -2070,7 +2043,7 @@ static SColumnNode* createColumnFromSmaFunc(uint64_t tableId, int32_t index, SEx strcpy(pCol->tableAlias, SMA_TABLE_NAME); pCol->node.resType = pSmaFunc->resType; strcpy(pCol->node.aliasName, pSmaFunc->aliasName); - return pCol; + return TSDB_CODE_SUCCESS; } static int32_t collectSmaFunc(SSmaIndexMatchFuncsCxt* pCxt, int32_t index, SNode* pSmaFunc) { @@ -2262,25 +2235,6 @@ static int32_t attemptApplySmaIndexImpl(STranslateContext* pCxt, SSelectStmt* pS return TSDB_CODE_SUCCESS; } -static void destroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } - -static int32_t attemptApplySmaIndex(STranslateContext* pCxt, SSelectStmt* pSelect) { - SRealTableNode* pRealTable = (SRealTableNode*)pSelect->pFromTable; - SArray* pIndexes = NULL; - int32_t code = getTableIndex(pCxt, pRealTable->table.dbName, pRealTable->table.tableName, &pIndexes); - if (TSDB_CODE_SUCCESS == code) { - code = attemptApplySmaIndexImpl(pCxt, pSelect, pIndexes); - } - taosArrayDestroyEx(pIndexes, destroySmaIndex); - return code; -} - -static int32_t attemptApplyIndex(STranslateContext* pCxt, SSelectStmt* pSelect) { - // if (mayBeApplySmaIndex(pSelect)) { - // return attemptApplySmaIndex(pCxt, pSelect); - // } - return TSDB_CODE_SUCCESS; -} #endif static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 41bb0481ad..5bde056b5c 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -807,6 +807,42 @@ int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFun return code; } -int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes) { - return TSDB_CODE_PAR_INTERNAL_ERROR; +static void destroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } + +static SArray* smaIndexesDup(SArray* pSrc) { + SArray* pDst = taosArrayDup(pSrc); + if (NULL == pDst) { + return NULL; + } + int32_t size = taosArrayGetSize(pDst); + for (int32_t i = 0; i < size; ++i) { + ((STableIndexInfo*)taosArrayGet(pDst, i))->expr = NULL; + } + for (int32_t i = 0; i < size; ++i) { + STableIndexInfo* pIndex = taosArrayGet(pDst, i); + pIndex->expr = taosMemoryStrDup(((STableIndexInfo*)taosArrayGet(pSrc, i))->expr); + if (NULL == pIndex->expr) { + taosArrayDestroyEx(pDst, destroySmaIndex); + return NULL; + } + } + return pDst; +} + +int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { + return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableIndex); +} + +int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes) { + char fullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pName, fullName); + SArray* pSmaIndexes = NULL; + int32_t code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableIndex, (void**)&pSmaIndexes); + if (TSDB_CODE_SUCCESS == code) { + *pIndexes = smaIndexesDup(pSmaIndexes); + if (NULL == *pIndexes) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + return code; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index c7f0990132..0b2aacf5ef 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -214,6 +214,11 @@ int32_t __catalogRefreshGetTableMeta(SCatalog* pCatalog, void* pTransporter, con int32_t __catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { return 0; } +int32_t __catalogGetTableIndex(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SName* pName, + SArray** pRes) { + return g_mockCatalogService->catalogGetTableIndex(pName, pRes); +} + void initMetaDataEnv() { g_mockCatalogService.reset(new MockCatalogService()); @@ -230,6 +235,7 @@ void initMetaDataEnv() { stub.set(catalogGetUdfInfo, __catalogGetUdfInfo); stub.set(catalogRefreshGetTableMeta, __catalogRefreshGetTableMeta); stub.set(catalogRemoveTableMeta, __catalogRemoveTableMeta); + stub.set(catalogGetTableIndex, __catalogGetTableIndex); // { // AddrAny any("libcatalog.so"); // std::map result; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 0c37c875c0..c730653e5e 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -149,6 +149,20 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } + int32_t catalogGetTableIndex(const SName* pTableName, SArray** pIndexes) const { + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(pTableName, tbFName); + auto it = index_.find(tbFName); + if (index_.end() == it) { + return TSDB_CODE_SUCCESS; + } + *pIndexes = taosArrayInit(it->second.size(), sizeof(STableIndexInfo)); + for (const auto& index : it->second) { + taosArrayPush(*pIndexes, &index); + } + return TSDB_CODE_SUCCESS; + } + int32_t catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const { int32_t code = getAllTableMeta(pCatalogReq->pTableMeta, &pMetaData->pTableMeta); if (TSDB_CODE_SUCCESS == code) { @@ -176,7 +190,7 @@ class MockCatalogServiceImpl { int32_t numOfColumns, int32_t numOfTags) { builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); meta_[db][tbname] = builder_->table(); - meta_[db][tbname]->schema->uid = id_++; + meta_[db][tbname]->schema->uid = getNextId(); return *(builder_.get()); } @@ -187,14 +201,11 @@ class MockCatalogServiceImpl { } meta_[db][tbname].reset(new MockTableMeta()); meta_[db][tbname]->schema = table.release(); - meta_[db][tbname]->schema->uid = id_++; + meta_[db][tbname]->schema->uid = getNextId(); meta_[db][tbname]->schema->tableType = TSDB_CHILD_TABLE; SVgroupInfo vgroup = {vgid, 0, 0, {0}, 0}; - addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); - addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030); - addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030); - vgroup.epSet.inUse = 0; + genEpSet(&vgroup.epSet); meta_[db][tbname]->vgs.emplace_back(vgroup); // super table @@ -268,10 +279,39 @@ class MockCatalogServiceImpl { udf_.insert(std::make_pair(func, info)); } + void createSmaIndex(const SMCreateSmaReq* pReq) { + STableIndexInfo info; + info.intervalUnit = pReq->intervalUnit; + info.slidingUnit = pReq->slidingUnit; + info.interval = pReq->interval; + info.offset = pReq->offset; + info.sliding = pReq->sliding; + info.dstTbUid = getNextId(); + info.dstVgId = pReq->dstVgId; + genEpSet(&info.epSet); + info.expr = strdup(pReq->expr); + auto it = index_.find(pReq->stb); + if (index_.end() == it) { + index_.insert(std::make_pair(std::string(pReq->stb), std::vector{info})); + } else { + it->second.push_back(info); + } + } + private: typedef std::map> TableMetaCache; typedef std::map DbMetaCache; typedef std::map> UdfMetaCache; + typedef std::map> IndexMetaCache; + + uint64_t getNextId() { return id_++; } + + void genEpSet(SEpSet* pEpSet) { + addEpIntoEpSet(pEpSet, "dnode_1", 6030); + addEpIntoEpSet(pEpSet, "dnode_2", 6030); + addEpIntoEpSet(pEpSet, "dnode_3", 6030); + pEpSet->inUse = 0; + } std::string toDbname(const std::string& dbFullName) const { std::string::size_type n = dbFullName.find("."); @@ -467,6 +507,7 @@ class MockCatalogServiceImpl { std::unique_ptr builder_; DbMetaCache meta_; UdfMetaCache udf_; + IndexMetaCache index_; }; MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) {} @@ -490,6 +531,8 @@ void MockCatalogService::createFunction(const std::string& func, int8_t funcType impl_->createFunction(func, funcType, outputType, outputLen, bufSize); } +void MockCatalogService::createSmaIndex(const SMCreateSmaReq* pReq) { impl_->createSmaIndex(pReq); } + int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const { return impl_->catalogGetTableMeta(pTableName, pTableMeta); } @@ -510,6 +553,10 @@ int32_t MockCatalogService::catalogGetUdfInfo(const std::string& funcName, SFunc return impl_->catalogGetUdfInfo(funcName, pInfo); } +int32_t MockCatalogService::catalogGetTableIndex(const SName* pTableName, SArray** pIndexes) const { + return impl_->catalogGetTableIndex(pTableName, pIndexes); +} + int32_t MockCatalogService::catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const { return impl_->catalogGetAllMeta(pCatalogReq, pMetaData); } diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index fafad81e95..c4ab091b7a 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -64,6 +64,7 @@ class MockCatalogService { int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const; int32_t catalogGetDBVgInfo(const char* pDbFName, SArray** pVgList) const; int32_t catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const; + int32_t catalogGetTableIndex(const SName* pTableName, SArray** pIndexes) const; int32_t catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const; private: diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index f252fe7265..76338eb491 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -32,8 +32,7 @@ typedef struct SOptimizeContext { bool optimized; } SOptimizeContext; -typedef int32_t (*FMatch)(SOptimizeContext* pCxt, SLogicNode* pLogicNode); -typedef int32_t (*FOptimize)(SOptimizeContext* pCxt, SLogicNode* pLogicNode); +typedef int32_t (*FOptimize)(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan); typedef struct SOptimizeRule { char* pName; @@ -109,7 +108,6 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { } if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) { return true; - // return (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType); } return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); } @@ -231,9 +229,9 @@ static void setScanWindowInfo(SScanLogicNode* pScan) { } } -static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { +static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SOsdInfo info = {0}; - int32_t code = osdMatch(pCxt, pLogicNode, &info); + int32_t code = osdMatch(pCxt, pLogicSubplan->pNode, &info); if (TSDB_CODE_SUCCESS == code && info.pScan) { setScanWindowInfo((SScanLogicNode*)info.pScan); } @@ -635,8 +633,8 @@ static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) return code; } -static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { - return cpdPushCondition(pCxt, pLogicNode); +static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + return cpdPushCondition(pCxt, pLogicSubplan->pNode); } static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) { @@ -745,26 +743,142 @@ static int32_t opkOptimizeImpl(SOptimizeContext* pCxt, SSortLogicNode* pSort) { return code; } -static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { - SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicNode, opkSortMayBeOptimized); +static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, opkSortMayBeOptimized); if (NULL == pSort) { return TSDB_CODE_SUCCESS; } return opkOptimizeImpl(pCxt, pSort); } -static const SOptimizeRule optimizeRuleSet[] = {{.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, - {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, - {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}}; +static int32_t smaOptFindSmaFunc(SNodeList* pSmaFuncs, SNode* pFunc, SNode** pCol, bool* pFound) { + int32_t index = 0; + SNode* pSmaFunc = NULL; + FOREACH(pSmaFunc, pSmaFuncs) { + if (nodesEqualNode(pSmaFunc, pFunc)) { + *pFound = true; + return collectSmaFunc(pCxt, index, pSmaFunc); + } + ++index; + } + return TSDB_CODE_SUCCESS; +} + +static bool smaOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex) { + if (pScan->interval != pIndex->interval || pScan->intervalUnit != pIndex->intervalUnit || + pScan->offset != pIndex->offset || pScan->sliding != pIndex->sliding || + pScan->slidingUnit != pIndex->slidingUnit) { + return false; + } + // todo time range + SNodeList* pFuncs = NULL; + SNode* pFunc = NULL; + FOREACH(pFunc, ((SWindowLogicNode*)pScan->node.pParent)->pFuncs) {} + + return true; +} + +static bool smaOptMayBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) { + return false; + } + + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + if (0 == pScan->interval || NULL == pScan->pSmaIndexes || NULL != pScan->node.pConditions) { + return false; + } + + int32_t size = taosArrayGetSize(pScan->pSmaIndexes); + for (int32_t i = 0; i < size; ++i) { + STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); + } + + return false; +} + +static int32_t smaOptCreateMerge(SNodeList* pTargets) { + SMergeLogicNode* pMerge = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); + if (NULL == pMerge) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pMerge->node.precision = pPartChild->precision; + pMerge->pMergeKeys = pMergeKeys; + + int32_t code = TSDB_CODE_SUCCESS; + pMerge->pInputs = nodesCloneList(pPartChild->pTargets); + pMerge->node.pTargets = nodesCloneList(pTargets); + if (NULL == pMerge->node.pTargets || NULL == pMerge->pInputs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + if (NULL == pSubplan) { + code = nodesListMakeAppend(&pSplitNode->pChildren, pMerge); + } else { + code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); + } + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pMerge); + } + return code; +} + +static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols, + SScanLogicNode** pOutput) { + SScanLogicNode* pSmaScan = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); + if (NULL == pSmaScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pSmaScan->pScanCols = pCols; + pSmaScan->tableType = TSDB_SUPER_TABLE; + pSmaScan->tableId = pIndex->dstTbUid; + pSmaScan->stableId = pIndex->dstTbUid; + pSmaScan->scanType = SCAN_TYPE_TABLE; + pSmaScan->scanSeq[0] = pScan->scanSeq[0]; + pSmaScan->scanSeq[1] = pScan->scanSeq[1]; + pSmaScan->scanRange = pScan->scanRange; + pSmaScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; + + pSmaScan->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + if (NULL == pSmaScan->pVgroupList) { + nodesDestroyNode(pSmaScan); + return TSDB_CODE_OUT_OF_MEMORY; + } + pSmaScan->pVgroupList->numOfVgroups = 1; + pSmaScan->pVgroupList->vgroups[0].vgId = pIndex->dstVgId; + memcpy(&(pSmaScan->pVgroupList->vgroups[0].epSet), &pIndex->epSet, sizeof(SEpSet)); + + *pOutput = pSmaScan; + return TSDB_CODE_SUCCESS; +} + +static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SScanLogicNode* pScan) { return TSDB_CODE_SUCCESS; } + +static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { + SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, opkSortMayBeOptimized); + if (NULL == pScan) { + return TSDB_CODE_SUCCESS; + } + return smaOptimizeImpl(pCxt, pScan); +} + +// clang-format off +static const SOptimizeRule optimizeRuleSet[] = { + {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, + {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, + {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, + {.pName = "SmaIndex", .optimizeFunc = smaOptimize} +}; +// clang-format on static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule)); -static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicNode* pLogicNode) { +static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { SOptimizeContext cxt = {.pPlanCxt = pCxt, .optimized = false}; do { cxt.optimized = false; for (int32_t i = 0; i < optimizeRuleNum; ++i) { - int32_t code = optimizeRuleSet[i].optimizeFunc(&cxt, pLogicNode); + int32_t code = optimizeRuleSet[i].optimizeFunc(&cxt, pLogicSubplan); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -774,5 +888,5 @@ static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicNode* pLogicNode) { } int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { - return applyOptimizeRule(pCxt, pLogicSubplan->pNode); + return applyOptimizeRule(pCxt, pLogicSubplan); } diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index 04d76a8b91..85f8b7d9f6 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -43,6 +43,8 @@ TEST_F(PlanOtherTest, createSmaIndex) { useDb("root", "test"); run("CREATE SMA INDEX idx1 ON t1 FUNCTION(MAX(c1), MIN(c3 + 10), SUM(c4)) INTERVAL(10s)"); + + run("SELECT SUM(c4) FROM t1 INTERVAL(10s)"); } TEST_F(PlanOtherTest, explain) { diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 4e89f9d008..57d7cb6608 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -14,12 +14,14 @@ */ #include "planTestUtil.h" + #include #include #include #include "cmdnodes.h" +#include "mockCatalogService.h" #include "parser.h" #include "planInt.h" @@ -361,6 +363,7 @@ class PlannerTestBaseImpl { } else if (QUERY_NODE_CREATE_INDEX_STMT == nodeType(pQuery->pRoot)) { SMCreateSmaReq req = {0}; tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); + g_mockCatalogService->createSmaIndex(&req); nodesStringToNode(req.ast, &pCxt->pAstRoot); pCxt->streamQuery = true; } else if (QUERY_NODE_CREATE_STREAM_STMT == nodeType(pQuery->pRoot)) { diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index b15c188f04..45a2ffec77 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE + #include "tjson.h" #include "cJSON.h" #include "taoserror.h" @@ -138,6 +139,23 @@ int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* return TSDB_CODE_SUCCESS; } +int32_t tjsonAddTArray(SJson* pJson, const char* pName, FToJson func, const SArray* pArray) { + int32_t num = taosArrayGetSize(pArray); + if (num > 0) { + SJson* pJsonArray = tjsonAddArrayToObject(pJson, pName); + if (NULL == pJsonArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; i < num; ++i) { + int32_t code = tjsonAddItem(pJsonArray, func, taosArrayGet(pArray, i)); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } + return TSDB_CODE_SUCCESS; +} + char* tjsonToString(const SJson* pJson) { return cJSON_Print((cJSON*)pJson); } char* tjsonToUnformattedString(const SJson* pJson) { return cJSON_PrintUnformatted((cJSON*)pJson); } @@ -184,7 +202,7 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal return TSDB_CODE_FAILED; } #ifdef WINDOWS - sscanf(p,"%lld",pVal); + sscanf(p, "%lld", pVal); #else // sscanf(p,"%ld",pVal); *pVal = taosStr2Int64(p, NULL, 10); @@ -219,7 +237,7 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV return TSDB_CODE_FAILED; } #ifdef WINDOWS - sscanf(p,"%llu",pVal); + sscanf(p, "%llu", pVal); #else // sscanf(p,"%ld",pVal); *pVal = taosStr2UInt64(p, NULL, 10); @@ -299,24 +317,43 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void return TSDB_CODE_SUCCESS; } +int32_t tjsonToTArray(const SJson* pJson, const char* pName, FToObject func, SArray** pArray, int32_t itemSize) { + const cJSON* jArray = tjsonGetObjectItem(pJson, pName); + int32_t size = tjsonGetArraySize(jArray); + if (size > 0) { + *pArray = taosArrayInit(size, itemSize); + if (NULL == *pArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosArraySetSize(*pArray, size); + for (int32_t i = 0; i < size; ++i) { + int32_t code = func(tjsonGetArrayItem(jArray, i), taosArrayGet(*pArray, i)); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } + return TSDB_CODE_SUCCESS; +} + SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); } -bool tjsonValidateJson(const char *jIn) { - if (!jIn){ +bool tjsonValidateJson(const char* jIn) { + if (!jIn) { return false; } // set json real data - cJSON *root = cJSON_Parse(jIn); - if (root == NULL){ + cJSON* root = cJSON_Parse(jIn); + if (root == NULL) { return false; } - if(!cJSON_IsObject(root)){ + if (!cJSON_IsObject(root)) { return false; } int size = cJSON_GetArraySize(root); - for(int i = 0; i < size; i++) { + for (int i = 0; i < size; i++) { cJSON* item = cJSON_GetArrayItem(root, i); if (!item) { return false; From 62780bbfb4cbd4432e49b42e646435562db6f242 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 10 Jun 2022 10:24:52 +0800 Subject: [PATCH 05/17] feature: add merge interval operator --- source/libs/executor/src/timewindowoperator.c | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 3ed416ec7c..842e46a8cd 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3236,10 +3236,6 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval, iaInfo->interval.precision, &iaInfo->win); - //TODO: pBlock not process not finished - //TODO: different block group id or no group id - //TODO: the last datablock - //TODO: blockDataUpdateTsWindow(pBlock, 0); int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, iaInfo->binfo.pCtx, @@ -3370,7 +3366,6 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { if (pRes->info.rows == 0) { doSetOperatorCompleted(pOperator); } else { - //TODO: ts column index blockDataUpdateTsWindow(pRes, 0); } @@ -3387,36 +3382,38 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI if (miaInfo == NULL || pOperator == NULL) { goto _error; } - SIntervalAggOperatorInfo *pInfo = &miaInfo->intervalAggOperatorInfo; - pInfo->win = pTaskInfo->window; - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; - pInfo->execModel = pTaskInfo->execModel; - pInfo->twAggSup = *pTwAggSupp; + SIntervalAggOperatorInfo * iaInfo = &miaInfo->intervalAggOperatorInfo; - pInfo->primaryTsIndex = primaryTsSlotId; + iaInfo->win = pTaskInfo->window; + iaInfo->order = TSDB_ORDER_ASC; + iaInfo->interval = *pInterval; + + iaInfo->execModel = pTaskInfo->execModel; + iaInfo->twAggSup = *pTwAggSupp; + + iaInfo->primaryTsIndex = primaryTsSlotId; miaInfo->groupIntervalHash = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, 4096); int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + initAggInfo(&iaInfo->binfo, &iaInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win); - pInfo->timeWindowInterpo = timeWindowinterpNeeded(pInfo->binfo.pCtx, numOfCols, pInfo); - if (pInfo->timeWindowInterpo) { - pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition)); + iaInfo->timeWindowInterpo = timeWindowinterpNeeded(iaInfo->binfo.pCtx, numOfCols, iaInfo); + if (iaInfo->timeWindowInterpo) { + iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition)); } - // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); - if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { + // iaInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || iaInfo->pTableQueryInfo == NULL*/) { goto _error; } - initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + initResultRowInfo(&iaInfo->binfo.resultRowInfo, (int32_t)1); pOperator->name = "TimeMergeIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; @@ -3425,10 +3422,10 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI pOperator->pExpr = pExprInfo; pOperator->pTaskInfo = pTaskInfo; pOperator->numOfExprs = numOfCols; - pOperator->info = pInfo; + pOperator->info = miaInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL, - destroyIntervalOperatorInfo, NULL, NULL, NULL); + destroyMergeIntervalOperatorInfo, NULL, NULL, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -3438,8 +3435,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI return pOperator; _error: - destroyMergeIntervalOperatorInfo(pInfo, numOfCols); - taosMemoryFreeClear(pInfo); + destroyMergeIntervalOperatorInfo(miaInfo, numOfCols); + taosMemoryFreeClear(miaInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; From b3ce29dc4aae00bd3a99b3f3328fe46f63b81718 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 10 Jun 2022 11:09:31 +0800 Subject: [PATCH 06/17] add time window operator --- source/libs/executor/inc/executorimpl.h | 3 + source/libs/executor/src/executorimpl.c | 24 ++- source/libs/executor/src/timewindowoperator.c | 138 +++++++++--------- 3 files changed, 90 insertions(+), 75 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 6a7e71412e..969a3aac62 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -797,6 +797,9 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e82c94073c..480814acef 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1956,8 +1956,9 @@ static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_ } int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, - SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, - SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, + const int32_t* rowCellOffset, SSDataBlock* pBlock, + SExecTaskInfo* pTaskInfo) { SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); SResultRow* pRow = (SResultRow*)((char*)page + resultRowPosition->offset); @@ -4553,7 +4554,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STimeWindowAggSupp twSup = { - .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; + .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; tsdbReaderT pDataReader = NULL; if (pHandle->vnode) { pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); @@ -4665,6 +4666,21 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) { + SMergeIntervalPhysiNode * pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; + + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + + SInterval interval = {.interval = pIntervalPhyNode->interval, + .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, + .offset = pIntervalPhyNode->offset, + .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; + + int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; + pOptr = createMergeIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL == type) { int32_t children = 8; pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); @@ -4697,7 +4713,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t numOfOutputCols = 0; SArray* pColList = extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID); - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); + SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); SSDataBlock* pInputDataBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc); pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pInputDataBlock, pResBlock, sortInfo, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION == type) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 842e46a8cd..32c3b84527 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1870,8 +1870,8 @@ _error: return NULL; } -static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, - int32_t tableGroupId, SArray* pUpdated) { +static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId, + SArray* pUpdated) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -1886,7 +1886,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; } else { - return ; + return; } int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); @@ -1903,13 +1903,14 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc pos->groupId = tableGroupId; pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; *(int64_t*)pos->key = pResult->win.skey; - forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, - nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) { - saveResult(pResult, tableGroupId, pUpdated); + saveResult(pResult, tableGroupId, pUpdated); } // window start(end) key interpolation - // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardRows); + // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, + // forwardRows); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); @@ -2040,10 +2041,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SIntervalAggOperatorInfo* pChildInfo = pChildOp->info; - doClearWindows(&pChildInfo->aggSup, &pChildInfo->binfo, &pChildInfo->interval, - pChildInfo->primaryTsIndex, pChildOp->numOfExprs, pBlock, NULL); - rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId, - pOperator->numOfExprs, pOperator->pTaskInfo); + doClearWindows(&pChildInfo->aggSup, &pChildInfo->binfo, &pChildInfo->interval, pChildInfo->primaryTsIndex, + pChildOp->numOfExprs, pBlock, NULL); + rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->numOfExprs, + pOperator->pTaskInfo); taosArrayDestroy(pUpWins); continue; } @@ -2053,7 +2054,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } if (isFinalInterval(pInfo)) { - int32_t chIndex = getChildIndex(pBlock); + int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); // if chIndex + 1 - size > 0, add new child for (int32_t i = 0; i < chIndex + 1 - size; i++) { @@ -2063,7 +2064,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } taosArrayPush(pInfo->pChildren, &pChildOp); } - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; setInputDataBlock(pChildOp, pChInfo->binfo.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL); @@ -2071,12 +2072,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } - + if (isFinalInterval(pInfo)) { - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, - &pInfo->interval, pClosed); - finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pClosed, - pInfo->binfo.rowCellInfoOffset); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pClosed); + finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pClosed, pInfo->binfo.rowCellInfoOffset); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { taosArrayAddAll(pUpdated, pClosed); } @@ -2100,34 +2099,35 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->binfo.pRes; } -SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) { - SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; +SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, int32_t numOfChild) { + SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = (SInterval) {.interval = pIntervalPhyNode->interval, - .sliding = pIntervalPhyNode->sliding, - .intervalUnit = pIntervalPhyNode->intervalUnit, - .slidingUnit = pIntervalPhyNode->slidingUnit, - .offset = pIntervalPhyNode->offset, - .precision = - ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; - pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, + pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval, + .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, + .offset = pIntervalPhyNode->offset, + .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; + pInfo->twAggSup = (STimeWindowAggSupp){ + .waterMark = pIntervalPhyNode->window.watermark, .calTrigger = pIntervalPhyNode->window.triggerType, .maxTs = INT64_MIN, - .winMap = NULL, }; + .winMap = NULL, + }; pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, 4096); - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, - pResBlock, keyBufSize, pTaskInfo->id.str); + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -2149,7 +2149,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, if (!isFinalInterval(pInfo)) { pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; } - pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);\ + pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); pInfo->pUpdateRes->info.type = STREAM_REPROCESS; blockDataEnsureCapacity(pInfo->pUpdateRes, 128); pInfo->pPhyNode = nodesCloneNode(pPhyNode); @@ -2163,9 +2163,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->numOfExprs = numOfCols; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, - destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, - NULL); + pOperator->fpSet = + createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2205,8 +2205,7 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { } } -int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, - int32_t numOfCols, SSDataBlock* pResultBlock) { +int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock) { pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset); pBasicInfo->pRes = pResultBlock; for (int32_t i = 0; i < numOfCols; ++i) { @@ -3179,10 +3178,10 @@ _error: typedef struct SMergeIntervalAggOperatorInfo { SIntervalAggOperatorInfo intervalAggOperatorInfo; - SHashObj* groupIntervalHash; - bool hasGroupId; - uint64_t groupId; - SSDataBlock *prefetchedBlock; + SHashObj* groupIntervalHash; + bool hasGroupId; + uint64_t groupId; + SSDataBlock* prefetchedBlock; } SMergeIntervalAggOperatorInfo; void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { @@ -3191,13 +3190,14 @@ void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); } -static int32_t outputPrevIntervalResult(SOperatorInfo * pOperatorInfo, uint64_t tableGroupId, SSDataBlock *pResultBlock, STimeWindow* newWin) { - SMergeIntervalAggOperatorInfo *miaInfo = pOperatorInfo->info; - SIntervalAggOperatorInfo * iaInfo = &miaInfo->intervalAggOperatorInfo; - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); +static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock, + STimeWindow* newWin) { + SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); - STimeWindow *prevWin= taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); + STimeWindow* prevWin = taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); if (prevWin == NULL) { taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); return 0; @@ -3205,8 +3205,8 @@ static int32_t outputPrevIntervalResult(SOperatorInfo * pOperatorInfo, uint64_t if (ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey) { SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &prevWin->skey, TSDB_KEYSIZE, tableGroupId); - SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, + GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); ASSERT(p1 != NULL); finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, iaInfo->binfo.pCtx, pOperatorInfo->pExpr, @@ -3220,9 +3220,9 @@ static int32_t outputPrevIntervalResult(SOperatorInfo * pOperatorInfo, uint64_t } static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, - int32_t scanFlag, SSDataBlock* pResultBlock) { - SMergeIntervalAggOperatorInfo *miaInfo = pOperatorInfo->info; - SIntervalAggOperatorInfo * iaInfo = &miaInfo->intervalAggOperatorInfo; + int32_t scanFlag, SSDataBlock* pResultBlock) { + SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -3255,10 +3255,9 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos); // restore current time window - ret = - setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, - iaInfo->binfo.pCtx, - numOfOutput, iaInfo->binfo.rowCellInfoOffset, &iaInfo->aggSup, pTaskInfo); + ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + iaInfo->binfo.pCtx, numOfOutput, iaInfo->binfo.rowCellInfoOffset, &iaInfo->aggSup, + pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -3314,11 +3313,10 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* } static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info; - SIntervalAggOperatorInfo *iaInfo = &miaInfo->intervalAggOperatorInfo; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -3327,9 +3325,9 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { blockDataCleanup(pRes); SOperatorInfo* downstream = pOperator->pDownstream[0]; - int32_t scanFlag = MAIN_SCAN; + int32_t scanFlag = MAIN_SCAN; while (1) { - SSDataBlock *pBlock = NULL; + SSDataBlock* pBlock = NULL; if (miaInfo->prefetchedBlock == NULL) { pBlock = downstream->fpSet.getNextFn(downstream); } else { @@ -3341,7 +3339,6 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { break; } - if (!miaInfo->hasGroupId) { miaInfo->hasGroupId = true; miaInfo->groupId = pBlock->info.groupId; @@ -3375,22 +3372,21 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + SExecTaskInfo* pTaskInfo) { SMergeIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeIntervalAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (miaInfo == NULL || pOperator == NULL) { goto _error; } - SIntervalAggOperatorInfo * iaInfo = &miaInfo->intervalAggOperatorInfo; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; iaInfo->win = pTaskInfo->window; iaInfo->order = TSDB_ORDER_ASC; iaInfo->interval = *pInterval; iaInfo->execModel = pTaskInfo->execModel; - iaInfo->twAggSup = *pTwAggSupp; iaInfo->primaryTsIndex = primaryTsSlotId; miaInfo->groupIntervalHash = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); From 7a4534a06a17a99e55ecab1bed6350b67990a95d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 10 Jun 2022 11:20:00 +0800 Subject: [PATCH 07/17] feat: before merge origin 3.0 --- source/libs/executor/inc/executorimpl.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 969a3aac62..cc1287df27 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -797,9 +797,13 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); + + SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, SExecTaskInfo* pTaskInfo); + + SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, From c31eb5231a7f5d3a2cc89a404e813b8cf0adbd35 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 10 Jun 2022 16:53:05 +0800 Subject: [PATCH 08/17] feat: sma index optimize --- source/libs/parser/src/parTranslater.c | 54 +------ source/libs/planner/inc/planInt.h | 1 + source/libs/planner/src/planOptimizer.c | 195 +++++++++++++++++------- source/libs/planner/src/planSpliter.c | 21 +-- source/libs/planner/src/planUtil.c | 17 +++ 5 files changed, 164 insertions(+), 124 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c2fa1ffd22..8994556c67 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -25,9 +25,6 @@ #include "tglobal.h" #include "ttime.h" -#define SMA_TABLE_NAME "#sma_table" -#define SMA_COL_NAME_PREFIX "#sma_col_" - #define generateDealNodeErrMsg(pCxt, code, ...) \ (pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__), DEAL_RES_ERROR) @@ -2029,22 +2026,7 @@ typedef struct SSmaIndexMatchFuncsCxt { bool match; } SSmaIndexMatchFuncsCxt; -static int32_t smaOptCreateSmaCol(SNode* pSmaFunc, int32_t index, SNode** pOutput) { - SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - return TSDB_CODE_SUCCESS; - } - pCol->tableId = tableId; - pCol->tableType = TSDB_SUPER_TABLE; - pCol->colId = index + 2; // skip timestamp primary col - pCol->colType = COLUMN_TYPE_COLUMN; - snprintf(pCol->colName, sizeof(pCol->colName), SMA_COL_NAME_PREFIX "%d", pCol->colId); - strcpy(pCol->tableName, SMA_TABLE_NAME); - strcpy(pCol->tableAlias, SMA_TABLE_NAME); - pCol->node.resType = pSmaFunc->resType; - strcpy(pCol->node.aliasName, pSmaFunc->aliasName); - return TSDB_CODE_SUCCESS; -} + static int32_t collectSmaFunc(SSmaIndexMatchFuncsCxt* pCxt, int32_t index, SNode* pSmaFunc) { if (NULL == pCxt->pUseMap) { @@ -2072,22 +2054,6 @@ static int32_t collectSmaFunc(SSmaIndexMatchFuncsCxt* pCxt, int32_t index, SNode return TSDB_CODE_SUCCESS; } -static int32_t findSmaFunc(SSmaIndexMatchFuncsCxt* pCxt, SNode* pNode, bool* pFound) { - if (!isAggFunc(pNode)) { - return TSDB_CODE_SUCCESS; - } - - int32_t index = 0; - SNode* pSmaFunc = NULL; - FOREACH(pSmaFunc, pCxt->pSmaFuncs) { - if (nodesEqualNode(pSmaFunc, pNode)) { - *pFound = true; - return collectSmaFunc(pCxt, index, pSmaFunc); - } - ++index; - } - return TSDB_CODE_SUCCESS; -} static EDealRes matchSmaFuncsImpl(SNode* pNode, void* pContext) { SSmaIndexMatchFuncsCxt* pCxt = pContext; @@ -2132,23 +2098,7 @@ static int32_t matchSmaFuncs(SSelectStmt* pSelect, STableIndexInfo* pIndex, SNod return cxt.errCode; } -static int32_t couldApplySmaIndex(STranslateContext* pCxt, SSelectStmt* pSelect, STableIndexInfo* pIndex, - SNodeList** pFuncs, SNodeList** pCols) { - if (!equalIntervalWindow((SIntervalWindowNode*)pSelect->pWindow, pSelect->pWhere, pIndex)) { - return TSDB_CODE_SUCCESS; - } - SNodeList* pSmaFuncs = NULL; - int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); - if (TSDB_CODE_SUCCESS == code) { - pCxt->currClause = SQL_CLAUSE_SELECT; - code = translateExprList(pCxt, pSmaFuncs); - } - if (TSDB_CODE_SUCCESS == code) { - code = matchSmaFuncs(pSelect, pIndex, pSmaFuncs, pFuncs, pCols); - } - nodesDestroyList(pSmaFuncs); - return code; -} + typedef struct SSmaIndexRewriteFuncsCxt { int32_t errCode; diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 6cfef2a7fb..f017f63404 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -37,6 +37,7 @@ extern "C" { int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...); int32_t createColumnByRewriteExps(SNodeList* pExprs, SNodeList** pList); +int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew); int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan); int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 76338eb491..f80128b652 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -751,35 +751,10 @@ static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) return opkOptimizeImpl(pCxt, pSort); } -static int32_t smaOptFindSmaFunc(SNodeList* pSmaFuncs, SNode* pFunc, SNode** pCol, bool* pFound) { - int32_t index = 0; - SNode* pSmaFunc = NULL; - FOREACH(pSmaFunc, pSmaFuncs) { - if (nodesEqualNode(pSmaFunc, pFunc)) { - *pFound = true; - return collectSmaFunc(pCxt, index, pSmaFunc); - } - ++index; - } - return TSDB_CODE_SUCCESS; -} - -static bool smaOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex) { - if (pScan->interval != pIndex->interval || pScan->intervalUnit != pIndex->intervalUnit || - pScan->offset != pIndex->offset || pScan->sliding != pIndex->sliding || - pScan->slidingUnit != pIndex->slidingUnit) { - return false; - } - // todo time range - SNodeList* pFuncs = NULL; - SNode* pFunc = NULL; - FOREACH(pFunc, ((SWindowLogicNode*)pScan->node.pParent)->pFuncs) {} - - return true; -} - static bool smaOptMayBeOptimized(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) { + if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode) || NULL == pNode->pParent || + QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) || + WINDOW_TYPE_INTERVAL != ((SWindowLogicNode*)pNode->pParent)->winType) { return false; } @@ -788,43 +763,44 @@ static bool smaOptMayBeOptimized(SLogicNode* pNode) { return false; } - int32_t size = taosArrayGetSize(pScan->pSmaIndexes); - for (int32_t i = 0; i < size; ++i) { - STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); - } - - return false; + return true; } -static int32_t smaOptCreateMerge(SNodeList* pTargets) { +static int32_t smaOptCreateMerge(SLogicNode* pChild, SNodeList* pMergeKeys, SNodeList* pTargets, SLogicNode** pOutput) { SMergeLogicNode* pMerge = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; } - pMerge->node.precision = pPartChild->precision; + pMerge->node.precision = pChild->precision; pMerge->pMergeKeys = pMergeKeys; - int32_t code = TSDB_CODE_SUCCESS; - pMerge->pInputs = nodesCloneList(pPartChild->pTargets); + pMerge->pInputs = nodesCloneList(pChild->pTargets); pMerge->node.pTargets = nodesCloneList(pTargets); if (NULL == pMerge->node.pTargets || NULL == pMerge->pInputs) { - code = TSDB_CODE_OUT_OF_MEMORY; + nodesDestroyNode(pMerge); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pOutput = (SLogicNode*)pMerge; + return TSDB_CODE_SUCCESS; +} + +static int32_t smaOptRecombinationNode(SLogicSubplan* pLogicSubplan, SLogicNode* pInterval, SLogicNode* pMerge, + SLogicNode* pSmaScan) { + int32_t code = nodesListMakeAppend(&pMerge->pChildren, pInterval); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&pMerge->pChildren, pSmaScan); } if (TSDB_CODE_SUCCESS == code) { - if (NULL == pSubplan) { - code = nodesListMakeAppend(&pSplitNode->pChildren, pMerge); - } else { - code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); - } - } - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyNode(pMerge); + code = replaceLogicNode(pLogicSubplan, pInterval, pMerge); + pSmaScan->pParent = pMerge; + pInterval->pParent = pMerge; } return code; } static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols, - SScanLogicNode** pOutput) { + SLogicNode** pOutput) { SScanLogicNode* pSmaScan = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); if (NULL == pSmaScan) { return TSDB_CODE_OUT_OF_MEMORY; @@ -848,18 +824,131 @@ static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pInde pSmaScan->pVgroupList->vgroups[0].vgId = pIndex->dstVgId; memcpy(&(pSmaScan->pVgroupList->vgroups[0].epSet), &pIndex->epSet, sizeof(SEpSet)); - *pOutput = pSmaScan; + *pOutput = (SLogicNode*)pSmaScan; return TSDB_CODE_SUCCESS; } -static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SScanLogicNode* pScan) { return TSDB_CODE_SUCCESS; } +static bool smaOptEqualInterval(SWindowLogicNode* pWindow, STableIndexInfo* pIndex) { + if (pWindow->interval != pIndex->interval || pWindow->intervalUnit != pIndex->intervalUnit || + pWindow->offset != pIndex->offset || pWindow->sliding != pIndex->sliding || + pWindow->slidingUnit != pIndex->slidingUnit) { + return false; + } + // todo time range + return true; +} -static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { - SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, opkSortMayBeOptimized); +// #define SMA_TABLE_NAME "#sma_table" +// #define SMA_COL_NAME_PREFIX "#sma_col_" + +static SNode* smaOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t colId) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return NULL; + } + pCol->tableId = tableId; + pCol->tableType = TSDB_SUPER_TABLE; + pCol->colId = colId; + pCol->colType = COLUMN_TYPE_COLUMN; + snprintf(pCol->colName, sizeof(pCol->colName), "#sma_col_%d", pCol->colId); + // strcpy(pCol->tableName, SMA_TABLE_NAME); + // strcpy(pCol->tableAlias, SMA_TABLE_NAME); + pCol->node.resType = ((SExprNode*)pFunc)->resType; + strcpy(pCol->node.aliasName, ((SExprNode*)pFunc)->aliasName); + return (SNode*)pCol; +} + +static int32_t smaOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { + int32_t index = 0; + SNode* pSmaFunc = NULL; + FOREACH(pSmaFunc, pSmaFuncs) { + if (nodesEqualNode(pQueryFunc, pSmaFunc)) { + return index; + } + ++index; + } + return -1; +} + +static int32_t smaOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, + int32_t* pWStrartIndex) { + SNodeList* pCols = NULL; + SNode* pFunc = NULL; + int32_t code = TSDB_CODE_SUCCESS; + int32_t index = 0; + FOREACH(pFunc, pFuncs) { + if (FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pFunc)->funcType) { + *pWStrartIndex = index; + } + int32_t smaFuncIndex = smaOptFindSmaFunc(pFunc, pSmaFuncs); + if (smaFuncIndex < 0) { + break; + } else { + code = nodesListMakeStrictAppend(&pCols, smaOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 2)); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + ++index; + } + + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pCols; + } else { + nodesDestroyList(pCols); + } + + return code; +} + +static int32_t smaOptCouldApplyIndex(SWindowLogicNode* pWindow, STableIndexInfo* pIndex, SNodeList** pCols, + int32_t* pWStrartIndex) { + if (!smaOptEqualInterval(pWindow, pIndex)) { + return TSDB_CODE_SUCCESS; + } + SNodeList* pSmaFuncs = NULL; + int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = smaOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); + } + nodesDestroyList(pSmaFuncs); + return code; +} + +static int32_t smaOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, + SNodeList* pSmaCols) { + SLogicNode* pSmaScan = NULL; + SLogicNode* pMerge = NULL; + int32_t code = smaOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); + if (TSDB_CODE_SUCCESS == code) { + code = smaOptCreateMerge(pScan->node.pParent, NULL, pSmaCols, &pMerge); + } + if (TSDB_CODE_SUCCESS == code) { + code = smaOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); + } + return code; +} + +static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) { + int32_t nindexes = taosArrayGetSize(pScan->pSmaIndexes); + for (int32_t i = 0; i < nindexes; ++i) { + STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); + SNodeList* pSmaCols = NULL; + int32_t wstrartIndex = -1; + int32_t code = smaOptCouldApplyIndex((SWindowLogicNode*)pScan->node.pParent, pIndex, &pSmaCols, &wstrartIndex); + if (TSDB_CODE_SUCCESS == code && NULL != pSmaCols) { + code = smaOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols); + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, smaOptMayBeOptimized); if (NULL == pScan) { return TSDB_CODE_SUCCESS; } - return smaOptimizeImpl(pCxt, pScan); + return smaOptimizeImpl(pCxt, pLogicSubplan, pScan); } // clang-format off diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index aebc0c14f4..e00ccf3be3 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -80,29 +80,12 @@ static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pChild, SE return TSDB_CODE_SUCCESS; } -static int32_t splReplaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew) { - if (NULL == pOld->pParent) { - pSubplan->pNode = (SLogicNode*)pNew; - return TSDB_CODE_SUCCESS; - } - - SNode* pNode; - FOREACH(pNode, pOld->pParent->pChildren) { - if (nodesEqualNode(pNode, pOld)) { - REPLACE_NODE(pNew); - pNew->pParent = pOld->pParent; - return TSDB_CODE_SUCCESS; - } - } - return TSDB_CODE_PLAN_INTERNAL_ERROR; -} - static int32_t splCreateExchangeNodeForSubplan(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, ESubplanType subplanType) { SExchangeLogicNode* pExchange = NULL; int32_t code = splCreateExchangeNode(pCxt, pSplitNode, &pExchange); if (TSDB_CODE_SUCCESS == code) { - code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pExchange); + code = replaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pExchange); } if (TSDB_CODE_SUCCESS == code) { pSubplan->subplanType = subplanType; @@ -322,7 +305,7 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla if (NULL == pSubplan) { code = nodesListMakeAppend(&pSplitNode->pChildren, pMerge); } else { - code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); + code = replaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); } } if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index 63d31912f0..a4ca864015 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -85,3 +85,20 @@ int32_t createColumnByRewriteExps(SNodeList* pExprs, SNodeList** pList) { } return cxt.errCode; } + +int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew) { + if (NULL == pOld->pParent) { + pSubplan->pNode = (SLogicNode*)pNew; + return TSDB_CODE_SUCCESS; + } + + SNode* pNode; + FOREACH(pNode, pOld->pParent->pChildren) { + if (nodesEqualNode(pNode, pOld)) { + REPLACE_NODE(pNew); + pNew->pParent = pOld->pParent; + return TSDB_CODE_SUCCESS; + } + } + return TSDB_CODE_PLAN_INTERNAL_ERROR; +} From d3cf0041640b59b22affbf6b8fd1c71833444b52 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 10 Jun 2022 18:03:44 +0800 Subject: [PATCH 09/17] fix(tmq): false cache --- source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tqRead.c | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index e7a744748b..5a8564bfd1 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -58,7 +58,7 @@ struct STqReadHandle { SArray* pColIdList; // SArray int32_t cachedSchemaVer; - int64_t cachedSchemaUid; + int64_t cachedSchemaSuid; SSchemaWrapper* pSchemaWrapper; STSchema* pSchema; }; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 2ecaeff747..0c38d6442b 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -67,7 +67,7 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { pReadHandle->ver = -1; pReadHandle->pColIdList = NULL; pReadHandle->cachedSchemaVer = -1; - pReadHandle->cachedSchemaUid = -1; + pReadHandle->cachedSchemaSuid = -1; pReadHandle->pSchema = NULL; pReadHandle->pSchemaWrapper = NULL; pReadHandle->tbIdHash = NULL; @@ -130,7 +130,8 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p // TODO set to real sversion /*int32_t sversion = 1;*/ int32_t sversion = htonl(pHandle->pBlock->sversion); - if (pHandle->cachedSchemaVer != sversion || pHandle->cachedSchemaUid != pHandle->msgIter.suid) { + if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion || + pHandle->cachedSchemaSuid != pHandle->msgIter.suid) { pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); if (pHandle->pSchema == NULL) { tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table", @@ -150,7 +151,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p return -1; } pHandle->cachedSchemaVer = sversion; - pHandle->cachedSchemaUid = pHandle->msgIter.suid; + pHandle->cachedSchemaSuid = pHandle->msgIter.suid; } STSchema* pTschema = pHandle->pSchema; From c038848f72ed74e905fe2665499e9957c1f710e8 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 10 Jun 2022 18:33:24 +0800 Subject: [PATCH 10/17] feature: output results in merge-interval agg buf when exhausted input stream --- source/libs/executor/src/executorimpl.c | 1 + source/libs/executor/src/timewindowoperator.c | 87 +++++++++++-------- 2 files changed, 52 insertions(+), 36 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4634cc9f47..726be6d0a2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2002,6 +2002,7 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi } releaseBufPage(pBuf, page); + pBlock->info.rows += pRow->numOfRows; return 0; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index a3c01629d6..c1c504a400 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3202,6 +3202,7 @@ typedef struct SMergeIntervalAggOperatorInfo { bool hasGroupId; uint64_t groupId; SSDataBlock* prefetchedBlock; + bool inputBlocksFinished; } SMergeIntervalAggOperatorInfo; void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { @@ -3223,7 +3224,7 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t return 0; } - if (ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey) { + if (newWin == NULL || (ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey) ) { SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &prevWin->skey, TSDB_KEYSIZE, tableGroupId); SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); @@ -3233,9 +3234,13 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t pOperatorInfo->numOfExprs, iaInfo->binfo.rowCellInfoOffset, pResultBlock, pTaskInfo); taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); - - taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); + if (newWin == NULL) { + taosHashRemove(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); + } else { + taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); + } } + return 0; } @@ -3343,47 +3348,57 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { SSDataBlock* pRes = iaInfo->binfo.pRes; blockDataCleanup(pRes); + blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); - SOperatorInfo* downstream = pOperator->pDownstream[0]; - int32_t scanFlag = MAIN_SCAN; - while (1) { - SSDataBlock* pBlock = NULL; - if (miaInfo->prefetchedBlock == NULL) { - pBlock = downstream->fpSet.getNextFn(downstream); - } else { - pBlock = miaInfo->prefetchedBlock; - miaInfo->groupId = pBlock->info.groupId; + if (!miaInfo->inputBlocksFinished) { + SOperatorInfo* downstream = pOperator->pDownstream[0]; + int32_t scanFlag = MAIN_SCAN; + while (1) { + SSDataBlock* pBlock = NULL; + if (miaInfo->prefetchedBlock == NULL) { + pBlock = downstream->fpSet.getNextFn(downstream); + } else { + pBlock = miaInfo->prefetchedBlock; + miaInfo->groupId = pBlock->info.groupId; + } + + if (pBlock == NULL) { + miaInfo->inputBlocksFinished = true; + break; + } + + if (!miaInfo->hasGroupId) { + miaInfo->hasGroupId = true; + miaInfo->groupId = pBlock->info.groupId; + } else if (miaInfo->groupId != pBlock->info.groupId) { + miaInfo->prefetchedBlock = pBlock; + break; + } + + getTableScanInfo(pOperator, &iaInfo->order, &scanFlag); + setInputDataBlock(pOperator, iaInfo->binfo.pCtx, pBlock, iaInfo->order, scanFlag, true); + STableQueryInfo* pTableQueryInfo = iaInfo->pCurrent; + + setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); + doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); + + if (pRes->info.rows >= pOperator->resultInfo.threshold) { + break; + } } - if (pBlock == NULL) { - break; - } - - if (!miaInfo->hasGroupId) { - miaInfo->hasGroupId = true; - miaInfo->groupId = pBlock->info.groupId; - } else if (miaInfo->groupId != pBlock->info.groupId) { - miaInfo->prefetchedBlock = pBlock; - break; - } - - getTableScanInfo(pOperator, &iaInfo->order, &scanFlag); - setInputDataBlock(pOperator, iaInfo->binfo.pCtx, pBlock, iaInfo->order, scanFlag, true); - STableQueryInfo* pTableQueryInfo = iaInfo->pCurrent; - - setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); - - if (pRes->info.rows >= pOperator->resultInfo.threshold) { - break; + pRes->info.groupId = miaInfo->groupId; + } else { + void* p = taosHashIterate(miaInfo->groupIntervalHash, NULL); + if (p != NULL) { + size_t len = 0; + uint64_t* pKey = taosHashGetKey(p, &len); + outputPrevIntervalResult(pOperator, *pKey, pRes, NULL); } } - pRes->info.groupId = miaInfo->groupId; if (pRes->info.rows == 0) { doSetOperatorCompleted(pOperator); - } else { - blockDataUpdateTsWindow(pRes, 0); } size_t rows = pRes->info.rows; From be88d8027f9202e8c848a064029f3a7bd434fc3d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 10 Jun 2022 19:31:48 +0800 Subject: [PATCH 11/17] enh(query): refactor function merge code to provide common interface --- source/libs/function/src/builtinsimpl.c | 171 +++++++++++++----------- 1 file changed, 91 insertions(+), 80 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1eafd3c649..ff838eb9c9 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2103,8 +2103,49 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } +static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* pOutput) { + pOutput->percent = pInput->percent; + pOutput->algo = pInput->algo; + if (pOutput->algo == APERCT_ALGO_TDIGEST) { + buildTDigestInfo(pInput); + tdigestAutoFill(pInput->pTDigest, COMPRESSION); + + if(pInput->pTDigest->num_centroids == 0 && pInput->pTDigest->num_buffered_pts == 0) { + return; + } + + buildTDigestInfo(pOutput); + TDigest *pTDigest = pOutput->pTDigest; + + if(pTDigest->num_centroids <= 0) { + memcpy(pTDigest, pInput->pTDigest, (size_t)TDIGEST_SIZE(COMPRESSION)); + tdigestAutoFill(pTDigest, COMPRESSION); + } else { + tdigestMerge(pTDigest, pInput->pTDigest); + } + } else { + buildHistogramInfo(pInput); + if (pInput->pHisto->numOfElems <= 0) { + return; + } + + buildHistogramInfo(pOutput); + SHistogramInfo *pHisto = pOutput->pHisto; + + if (pHisto->numOfElems <= 0) { + memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); + } else { + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); + SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); + memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); + tHistogramDestroy(&pRes); + } + } +} + int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { - int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -2113,60 +2154,14 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - SAPercentileInfo* pInputInfo; int32_t start = pInput->startRowIndex; - for (int32_t i = start; i < pInput->numOfRows + start; ++i) { - //if (colDataIsNull_s(pCol, i)) { - // continue; - //} - numOfElems += 1; - char* data = colDataGetData(pCol, i); + char* data = colDataGetData(pCol, start); + SAPercentileInfo* pInputInfo = (SAPercentileInfo *)varDataVal(data); - pInputInfo = (SAPercentileInfo *)varDataVal(data); - } + apercentileTransferInfo(pInputInfo, pInfo); - pInfo->percent = pInputInfo->percent; - pInfo->algo = pInputInfo->algo; - if (pInfo->algo == APERCT_ALGO_TDIGEST) { - buildTDigestInfo(pInputInfo); - tdigestAutoFill(pInputInfo->pTDigest, COMPRESSION); - - if(pInputInfo->pTDigest->num_centroids == 0 && pInputInfo->pTDigest->num_buffered_pts == 0) { - return TSDB_CODE_SUCCESS; - } - - buildTDigestInfo(pInfo); - TDigest *pTDigest = pInfo->pTDigest; - - if(pTDigest->num_centroids <= 0) { - memcpy(pTDigest, pInputInfo->pTDigest, (size_t)TDIGEST_SIZE(COMPRESSION)); - tdigestAutoFill(pTDigest, COMPRESSION); - } else { - tdigestMerge(pTDigest, pInputInfo->pTDigest); - } - } else { - buildHistogramInfo(pInputInfo); - if (pInputInfo->pHisto->numOfElems <= 0) { - return TSDB_CODE_SUCCESS; - } - - buildHistogramInfo(pInfo); - SHistogramInfo *pHisto = pInfo->pHisto; - - if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInputInfo->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - } else { - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInputInfo->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - tHistogramDestroy(&pRes); - } - } - - SET_VAL(pResInfo, numOfElems, 1); + SET_VAL(pResInfo, 1, 1); return TSDB_CODE_SUCCESS; } @@ -3049,6 +3044,17 @@ _spread_over: return TSDB_CODE_SUCCESS; } +static void spreadTransferInfo(SSpreadInfo* pInput, SSpreadInfo* pOutput) { + pOutput->hasResult = pInput->hasResult; + if (pInput->max > pOutput->max) { + pOutput->max = pInput->max; + } + + if (pInput->min < pOutput->min) { + pOutput->min = pInput->min; + } +} + int32_t spreadFunctionMerge(SqlFunctionCtx *pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; @@ -3061,14 +3067,7 @@ int32_t spreadFunctionMerge(SqlFunctionCtx *pCtx) { char* data = colDataGetData(pCol, start); pInputInfo = (SSpreadInfo *)varDataVal(data); - pInfo->hasResult = pInputInfo->hasResult; - if (pInputInfo->max > pInfo->max) { - pInfo->max = pInputInfo->max; - } - - if (pInputInfo->min < pInfo->min) { - pInfo->min = pInputInfo->min; - } + spreadTransferInfo(pInputInfo, pInfo); SET_VAL(GET_RES_INFO(pCtx), 1, 1); @@ -3206,6 +3205,17 @@ _elapsed_over: return TSDB_CODE_SUCCESS; } +static void elapsedTransferInfo(SElapsedInfo* pInput, SElapsedInfo* pOutput) { + pOutput->timeUnit = pInput->timeUnit; + if (pOutput->min > pInput->min) { + pOutput->min = pInput->min; + } + + if (pOutput->max < pInput->max) { + pOutput->max = pInput->max; + } +} + int32_t elapsedFunctionMerge(SqlFunctionCtx *pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; @@ -3217,14 +3227,7 @@ int32_t elapsedFunctionMerge(SqlFunctionCtx *pCtx) { char* data = colDataGetData(pCol, start); SElapsedInfo* pInputInfo = (SElapsedInfo *)varDataVal(data); - pInfo->timeUnit = pInputInfo->timeUnit; - if (pInfo->min > pInputInfo->min) { - pInfo->min = pInputInfo->min; - } - - if (pInfo->max < pInputInfo->max) { - pInfo->max = pInputInfo->max; - } + elapsedTransferInfo(pInputInfo, pInfo); SET_VAL(GET_RES_INFO(pCtx), 1, 1); return TSDB_CODE_SUCCESS; @@ -3470,6 +3473,17 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } +static void histogramTransferInfo(SHistoFuncInfo* pInput, SHistoFuncInfo* pOutput) { + pOutput->normalized = pInput->normalized; + pOutput->numOfBins = pInput->numOfBins; + pOutput->totalCount += pInput->totalCount; + for (int32_t k = 0; k < pOutput->numOfBins; ++k) { + pOutput->bins[k].lower = pInput->bins[k].lower; + pOutput->bins[k].upper = pInput->bins[k].upper; + pOutput->bins[k].count += pInput->bins[k].count; + } +} + int32_t histogramFunctionMerge(SqlFunctionCtx *pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; @@ -3481,14 +3495,7 @@ int32_t histogramFunctionMerge(SqlFunctionCtx *pCtx) { char* data = colDataGetData(pCol, start); SHistoFuncInfo* pInputInfo = (SHistoFuncInfo *)varDataVal(data); - pInfo->normalized = pInputInfo->normalized; - pInfo->numOfBins = pInputInfo->numOfBins; - pInfo->totalCount += pInputInfo->totalCount; - for (int32_t k = 0; k < pInfo->numOfBins; ++k) { - pInfo->bins[k].lower = pInputInfo->bins[k].lower; - pInfo->bins[k].upper = pInputInfo->bins[k].upper; - pInfo->bins[k].count += pInputInfo->bins[k].count; - } + histogramTransferInfo(pInputInfo, pInfo); SET_VAL(GET_RES_INFO(pCtx), pInfo->numOfBins, pInfo->numOfBins); return TSDB_CODE_SUCCESS; @@ -3676,6 +3683,14 @@ int32_t hllFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } +static void hllTransferInfo(SHLLInfo* pInput, SHLLInfo* pOutput) { + for (int32_t k = 0; k < HLL_BUCKETS; ++k) { + if (pOutput->buckets[k] < pInput->buckets[k]) { + pOutput->buckets[k] = pInput->buckets[k]; + } + } +} + int32_t hllFunctionMerge(SqlFunctionCtx *pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; @@ -3687,11 +3702,7 @@ int32_t hllFunctionMerge(SqlFunctionCtx *pCtx) { char* data = colDataGetData(pCol, start); SHLLInfo* pInputInfo = (SHLLInfo *)varDataVal(data); - for (int32_t k = 0; k < HLL_BUCKETS; ++k) { - if (pInfo->buckets[k] < pInputInfo->buckets[k]) { - pInfo->buckets[k] = pInputInfo->buckets[k]; - } - } + hllTransferInfo(pInputInfo, pInfo); SET_VAL(GET_RES_INFO(pCtx), 1, 1); return TSDB_CODE_SUCCESS; From 3a999d4eeb21ced857bbc34b500a253fbd1a848a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 10 Jun 2022 20:15:57 +0800 Subject: [PATCH 12/17] fix: update taos-tools (#13699) for 3.0 --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index 717f5aaa5f..932da0f4ca 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 717f5aaa5f0a1b4d92bb2ae68858fec554fb5eda +Subproject commit 932da0f4cac013c2eded824d1d4d01cfa6168fa3 From a7fba4b22491c051a4ddd4c7a0ccfafe1e7f2c30 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 10 Jun 2022 21:57:59 +0800 Subject: [PATCH 13/17] refactor(stream): stream scheduler --- source/dnode/mnode/impl/inc/mndScheduler.h | 2 +- source/dnode/mnode/impl/src/mndScheduler.c | 214 ++++++++++++++++++++- source/dnode/mnode/impl/src/mndStream.c | 2 +- 3 files changed, 206 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndScheduler.h b/source/dnode/mnode/impl/inc/mndScheduler.h index 05aea3f68c..fa5e20ab6f 100644 --- a/source/dnode/mnode/impl/inc/mndScheduler.h +++ b/source/dnode/mnode/impl/inc/mndScheduler.h @@ -27,7 +27,7 @@ void mndCleanupScheduler(SMnode* pMnode); int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub); -int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); +int32_t mndScheduleStream1(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); int32_t mndConvertRSmaTask(const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, char** pStr, int32_t* pLen, double filesFactor); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index d07355dfa9..deb7382183 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -127,6 +127,61 @@ int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet return 0; } +int32_t mndAddSinkToTask(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, SStreamTask* pTask) { + pTask->dispatchType = TASK_DISPATCH__NONE; + // sink + if (pStream->createdBy == STREAM_CREATED_BY__SMA) { + pTask->sinkType = TASK_SINK__SMA; + pTask->smaSink.smaId = pStream->smaId; + } else { + pTask->sinkType = TASK_SINK__TABLE; + pTask->tbSink.stbUid = pStream->targetStbUid; + memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); + pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); + } + return 0; +} + +int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, SStreamTask* pTask) { + pTask->sinkType = TASK_SINK__NONE; + if (pStream->fixedSinkVgId == 0) { + pTask->dispatchType = TASK_DISPATCH__SHUFFLE; + pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; + SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); + ASSERT(pDb); + + if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { + sdbRelease(pMnode->pSdb, pDb); + + SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t sz = taosArrayGetSize(pVgs); + SArray* sinkLv = taosArrayGetP(pStream->tasks, 0); + int32_t sinkLvSize = taosArrayGetSize(sinkLv); + for (int32_t i = 0; i < sz; i++) { + SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i); + for (int32_t j = 0; j < sinkLvSize; j++) { + SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); + if (pLastLevelTask->nodeId == pVgInfo->vgId) { + pVgInfo->taskId = pLastLevelTask->taskId; + break; + } + } + } + } else { + pTask->dispatchType = TASK_DISPATCH__FIXED; + pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; + SArray* pArray = taosArrayGetP(pStream->tasks, 0); + // one sink only + ASSERT(taosArrayGetSize(pArray) == 1); + SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); + pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId; + pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; + pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; + } + } + return 0; +} + int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { int32_t msgLen; pTask->nodeId = pVgroup->vgId; @@ -139,6 +194,7 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SS terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } + ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE || pTask->sinkType != TASK_SINK__NONE); mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_STREAM_TASK_DEPLOY, pVgroup->vgId); return 0; } @@ -182,7 +238,7 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { return pVgroup; } -int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { +int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; void* pIter = NULL; SArray* tasks = taosArrayGetP(pStream->tasks, 0); @@ -234,7 +290,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p return 0; } -int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { +int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { ASSERT(pStream->fixedSinkVgId != 0); SArray* tasks = taosArrayGetP(pStream->tasks, 0); SStreamTask* pTask = tNewSStreamTask(pStream->uid); @@ -279,6 +335,146 @@ int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStr return 0; } +int32_t mndScheduleStream1(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { + SSdb* pSdb = pMnode->pSdb; + SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); + if (pPlan == NULL) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + ASSERT(pStream->vgNum == 0); + + int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); + ASSERT(totLevel <= 2); + pStream->tasks = taosArrayInit(totLevel, sizeof(void*)); + + bool hasExtraSink = false; + bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; + if (totLevel == 2 || externalTargetDB) { + SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); + taosArrayPush(pStream->tasks, &taskOneLevel); + // add extra sink + hasExtraSink = true; + if (pStream->fixedSinkVgId == 0) { + mndAddShuffleSinkTasksToStream(pMnode, pTrans, pStream); + } else { + mndAddFixedSinkTaskToStream(pMnode, pTrans, pStream); + } + } + if (totLevel > 1) { + SStreamTask* pFinalTask; + // inner plan + { + SArray* taskInnerLevel = taosArrayInit(0, sizeof(void*)); + taosArrayPush(pStream->tasks, &taskInnerLevel); + + SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 0); + SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); + ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE); + + pFinalTask = tNewSStreamTask(pStream->uid); + mndAddTaskToTaskSet(taskInnerLevel, pFinalTask); + // input + pFinalTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; + + // dispatch + mndAddDispatcherToInnerTask(pMnode, pTrans, pStream, pFinalTask); + + // exec + pFinalTask->execType = TASK_EXEC__PIPE; + SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->dbUid); + if (mndAssignTaskToVg(pMnode, pTrans, pFinalTask, plan, pVgroup) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; + } + } + + // source plan + SArray* taskSourceLevel = taosArrayInit(0, sizeof(void*)); + taosArrayPush(pStream->tasks, &taskSourceLevel); + + SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 1); + SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); + ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); + + void* pIter = NULL; + while (1) { + SVgObj* pVgroup; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pStream->dbUid) { + sdbRelease(pSdb, pVgroup); + continue; + } + SStreamTask* pTask = tNewSStreamTask(pStream->uid); + mndAddTaskToTaskSet(taskSourceLevel, pTask); + + // input + pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK; + + // add fixed vg dispatch + pTask->sinkType = TASK_SINK__NONE; + pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; + pTask->dispatchType = TASK_DISPATCH__FIXED; + + pTask->fixedEpDispatcher.taskId = pFinalTask->taskId; + pTask->fixedEpDispatcher.nodeId = pFinalTask->nodeId; + pTask->fixedEpDispatcher.epSet = pFinalTask->epSet; + + // exec + pTask->execType = TASK_EXEC__PIPE; + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; + } + } + } + + if (totLevel == 1) { + SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); + taosArrayPush(pStream->tasks, &taskOneLevel); + + SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 0); + ASSERT(LIST_LENGTH(inner->pNodeList) == 1); + SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); + ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); + + void* pIter = NULL; + while (1) { + SVgObj* pVgroup; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pStream->dbUid) { + sdbRelease(pSdb, pVgroup); + continue; + } + SStreamTask* pTask = tNewSStreamTask(pStream->uid); + mndAddTaskToTaskSet(taskOneLevel, pTask); + + // input + pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK; + + // sink or dispatch + if (hasExtraSink) { + mndAddDispatcherToInnerTask(pMnode, pTrans, pStream, pTask); + } else { + mndAddSinkToTask(pMnode, pTrans, pStream, pTask); + } + + // exec + pTask->execType = TASK_EXEC__PIPE; + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; + } + } + } + return 0; +} + int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); @@ -300,14 +496,15 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // add extra sink hasExtraSink = true; if (pStream->fixedSinkVgId == 0) { - mndAddShuffledSinkToStream(pMnode, pTrans, pStream); + mndAddShuffleSinkTasksToStream(pMnode, pTrans, pStream); } else { - mndAddFixedSinkToStream(pMnode, pTrans, pStream); + mndAddFixedSinkTaskToStream(pMnode, pTrans, pStream); } } for (int32_t level = 0; level < totLevel; level++) { - SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); + SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); + taosArrayPush(pStream->tasks, &taskOneLevel); SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, level); ASSERT(LIST_LENGTH(inner->pNodeList) == 1); @@ -357,18 +554,17 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { } // dispatch part - if (level == 0) { + if (level == 0 && !hasExtraSink) { pTask->dispatchType = TASK_DISPATCH__NONE; } else { // add fixed ep dispatcher int32_t lastLevel = level - 1; - ASSERT(lastLevel == 0); if (hasExtraSink) lastLevel++; + ASSERT(lastLevel == 0); SArray* pArray = taosArrayGetP(pStream->tasks, lastLevel); // one merge only ASSERT(taosArrayGetSize(pArray) == 1); SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); - /*pTask->dispatchMsgType = TDMT_VND_TASK_MERGE_EXEC;*/ pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; pTask->dispatchType = TASK_DISPATCH__FIXED; @@ -465,8 +661,6 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { } sdbRelease(pSdb, pVgroup); } - - taosArrayPush(pStream->tasks, &taskOneLevel); } #if 0 diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7abe9e3c0d..4a2c0a59a1 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -269,7 +269,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast return -1; } - if (mndScheduleStream(pMnode, pTrans, pStream) < 0) { + if (mndScheduleStream1(pMnode, pTrans, pStream) < 0) { mError("stream:%ld, schedule stream since %s", pStream->uid, terrstr()); return -1; } From ae5219dd6e7cbc03b43d421b421679cee68de42e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 10 Jun 2022 22:29:28 +0800 Subject: [PATCH 14/17] refactor(sync): add heartbeat log --- include/common/tmsgdef.h | 2 +- source/dnode/mnode/impl/src/mndMain.c | 4 ++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +++++- source/libs/sync/src/syncMain.c | 4 ++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 6fb9c75e6b..3b38362647 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -221,7 +221,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MON_QM_LOAD, "monitor-qload", NULL, NULL) TD_NEW_MSG_SEG(TDMT_SYNC_MSG) - TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timeout", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_PING, "sync-ping", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_PING_REPLY, "sync-ping-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST, "sync-client-request", NULL, NULL) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 27d13d66b6..d8a61461dd 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -403,6 +403,10 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { char logBuf[512] = {0}; char *syncNodeStr = sync2SimpleStr(pMgmt->sync); snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); + static int64_t mndTick = 0; + if (++mndTick % 10 == 1) { + mTrace("sync trace msg:%s, %s", TMSG_INFO(pMsg->msgType), syncNodeStr); + } syncRpcMsgLog2(logBuf, pMsg); taosMemoryFree(syncNodeStr); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index da3c8e7c93..51ed739693 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -305,6 +305,10 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { char logBuf[512] = {0}; char *syncNodeStr = sync2SimpleStr(pVnode->sync); snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); + static int64_t vndTick = 0; + if (++vndTick % 10 == 1) { + vTrace("sync trace msg:%s, %s", TMSG_INFO(pMsg->msgType), syncNodeStr); + } syncRpcMsgLog2(logBuf, pMsg); taosMemoryFree(syncNodeStr); @@ -902,4 +906,4 @@ static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, vo // 2. adjust hash range / compact / remove wals / rename vgroups // 3. reload sync return 0; -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 26dbf6c47a..0a19e16d5c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1066,7 +1066,7 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { int len = 256; char* s = (char*)taosMemoryMalloc(len); snprintf(s, len, - "syncNode2SimpleStr vgId:%d currentTerm:%lu, commitIndex:%ld, state:%d %s, isStandBy:%d, " + "syncNode: vgId:%d currentTerm:%lu, commitIndex:%ld, state:%d %s, isStandBy:%d, " "electTimerLogicClock:%lu, " "electTimerLogicClockUser:%lu, " "electTimerMS:%d, replicaNum:%d", @@ -1873,4 +1873,4 @@ SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId) } } return pSender; -} \ No newline at end of file +} From 58808db4cd48009a8849b4e0e9ede2b01dde86ef Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 11 Jun 2022 09:25:18 +0800 Subject: [PATCH 15/17] feat: sma index optimize --- include/libs/nodes/plannodes.h | 3 +-- source/libs/catalog/src/ctgAsync.c | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f7891ba6c2..1a3bfdbb04 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -304,7 +304,7 @@ typedef struct SDownstreamSourceNode { typedef struct SExchangePhysiNode { SPhysiNode node; - int32_t srcGroupId; // group id of datasource suplans + int32_t srcGroupId; // group id of datasource suplans bool singleChannel; SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhysiNode; @@ -438,7 +438,6 @@ typedef struct SQueryPlan { int32_t numOfSubplans; SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SExplainInfo explainInfo; - SArray* pPlaceholderValues; } SQueryPlan; void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 2574528d15..f61a3637ed 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -525,7 +525,7 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableHash, &res); + taosArrayPush(pJob->jobRes.pTableIndex, &res); return TSDB_CODE_SUCCESS; } @@ -875,7 +875,9 @@ int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf TSWAP(pTask->res, pTask->msgCtx.out); _return: - + if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { + code = TSDB_CODE_SUCCESS; + } ctgHandleTaskEnd(pTask, code); CTG_RET(code); From ec10e972db6a8361d06daebb5ada2c68c126a960 Mon Sep 17 00:00:00 2001 From: Bo Ding Date: Sat, 11 Jun 2022 09:23:45 +0800 Subject: [PATCH 16/17] Update go.mdx --- docs-en/14-reference/03-connector/go.mdx | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs-en/14-reference/03-connector/go.mdx b/docs-en/14-reference/03-connector/go.mdx index c1e85ae4eb..8a05f2d841 100644 --- a/docs-en/14-reference/03-connector/go.mdx +++ b/docs-en/14-reference/03-connector/go.mdx @@ -55,25 +55,27 @@ A "REST connection" is a connection between the application and the TDengine ins ### Pre-installation -* Install Go development environment (Go 1.14 and above, GCC 4.8.5 and above) -* If you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](/reference/connector#Install Client Driver) for specific steps +- Install Go development environment (Go 1.14 and above, GCC 4.8.5 and above) +- If you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](/reference/connector/#install-client-driver) for specific steps Configure the environment variables and check the command. -* ```go env`` -* ```gcc -v`` +* `go env` +* `gcc -v` ### Use go get to install -``go get -u github.com/taosdata/driver-go/v2@develop`` +``` +go get -u github.com/taosdata/driver-go/v2@develop +``` ### Manage with go mod 1. Initialize the project with the `go mod` command. - ``text + ```text go mod init taos-demo - ``` text + ``` 2. Introduce taosSql @@ -88,7 +90,7 @@ Configure the environment variables and check the command. ```text go mod tidy - ``` 4. + ``` 4. Run the program with `go run taos-demo` or compile the binary with the `go build` command. @@ -309,6 +311,7 @@ func main() { :::info This API is created successfully without checking permissions, but only when you execute a Query or Exec, and check if user/password/host/port is legal. + ::: * `func (db *DB) Exec(query string, args . .interface{}) (Result, error)` From a395b3e91a00bbe0640757019f5aa9c12b16e16e Mon Sep 17 00:00:00 2001 From: dingbo Date: Sat, 11 Jun 2022 09:58:28 +0800 Subject: [PATCH 17/17] docs: fix some links and format errors --- docs-en/14-reference/03-connector/csharp.mdx | 8 +-- docs-en/14-reference/03-connector/java.mdx | 55 +++++++++++--------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/docs-en/14-reference/03-connector/csharp.mdx b/docs-en/14-reference/03-connector/csharp.mdx index 5eb322cf91..2d1b62fe89 100644 --- a/docs-en/14-reference/03-connector/csharp.mdx +++ b/docs-en/14-reference/03-connector/csharp.mdx @@ -48,7 +48,7 @@ Please refer to [version support list](/reference/connector#version-support) * Install the [.NET SDK](https://dotnet.microsoft.com/download) * [Nuget Client](https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools) (optional installation) -* Install TDengine client driver, please refer to [Install client driver](/reference/connector#Install client driver) for details +* Install TDengine client driver, please refer to [Install client driver](/reference/connector/#install-client-driver) for details ### Install via dotnet CLI @@ -57,7 +57,7 @@ Please refer to [version support list](/reference/connector#version-support) You can reference the `TDengine.Connector` published in Nuget to the current project via the `dotnet` command under the path of the existing .NET project. -``` bash +``` dotnet add package TDengine.Connector ``` @@ -66,7 +66,7 @@ dotnet add package TDengine.Connector You can download TDengine's source code and directly reference the latest version of the TDengine.Connector library -```bash +``` git clone https://github.com/taosdata/TDengine.git cd TDengine/src/connector/C#/src/ cp -r TDengineDriver/ myProject @@ -79,7 +79,7 @@ dotnet add TDengineDriver/TDengineDriver.csproj ## Create a connection -``` C# +```csharp using TDengineDriver; namespace TDengineExample diff --git a/docs-en/14-reference/03-connector/java.mdx b/docs-en/14-reference/03-connector/java.mdx index 6c40f753be..ff15acf1a9 100644 --- a/docs-en/14-reference/03-connector/java.mdx +++ b/docs-en/14-reference/03-connector/java.mdx @@ -69,7 +69,7 @@ Before using Java Connector to connect to the database, the following conditions ### Install the connectors - + - [sonatype](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) - [mvnrepository](https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver) @@ -77,7 +77,7 @@ Before using Java Connector to connect to the database, the following conditions Add following dependency in the `pom.xml` file of your Maven project: -```xml-dtd +```xml com.taosdata.jdbc taos-jdbcdriver @@ -90,7 +90,7 @@ Add following dependency in the `pom.xml` file of your Maven project: You can build Java connector from source code after cloning the TDengine project: -```shell +``` git clone https://github.com/taosdata/taos-connector-jdbc.git cd taos-connector-jdbc mvn clean install -Dmaven.test.skip=true @@ -140,40 +140,43 @@ When you use a JDBC native connection to connect to a TDengine cluster, you can 1. Do not specify hostname and port in Java applications. -```java -public Connection getConn() throws Exception{ - Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; - Properties connProps = new Properties(); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - Connection conn = DriverManager.getConnection(jdbcUrl, connProps); - return conn; -} -``` + ```java + public Connection getConn() throws Exception{ + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + return conn; + } + ``` 2. specify the firstEp and the secondEp in the configuration file taos.cfg -```shell -# first fully qualified domain name (FQDN) for TDengine system -firstEp cluster_node1:6030 + ```shell + # first fully qualified domain name (FQDN) for TDengine system + firstEp cluster_node1:6030 -# second fully qualified domain name (FQDN) for TDengine system, for cluster only -secondEp cluster_node2:6030 + # second fully qualified domain name (FQDN) for TDengine system, for cluster only + secondEp cluster_node2:6030 -# default system charset -# charset UTF-8 + # default system charset + # charset UTF-8 -# system locale -# locale en_US.UTF-8 -``` + # system locale + # locale en_US.UTF-8 + ``` In the above example, JDBC uses the client's configuration file to establish a connection to a hostname `cluster_node1`, port 6030, and a database named `test`. When the firstEp node in the cluster fails, JDBC attempts to connect to the cluster using secondEp. In TDengine, as long as one node in firstEp and secondEp is valid, the connection to the cluster can be established normally. -> **Note**: The configuration file here refers to the configuration file on the machine where the application that calls the JDBC Connector is located, the default path is `/etc/taos/taos.cfg` on Linux, and the default path is `C://TDengine/cfg/taos.cfg` on Windows. +:::note +The configuration file here refers to the configuration file on the machine where the application that calls the JDBC Connector is located, the default path is `/etc/taos/taos.cfg` on Linux, and the default path is `C://TDengine/cfg/taos.cfg` on Windows. + +:::