From e1b1ae575bf2a8ff454633695a1d27af37e0a186 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 26 Dec 2022 14:39:00 +0800 Subject: [PATCH 1/4] fix: select tbname from system tables --- source/libs/executor/src/sysscanoperator.c | 183 ++++++++++++++------- source/libs/planner/src/planOptimizer.c | 3 + source/libs/planner/src/planSpliter.c | 16 +- source/libs/planner/src/planner.c | 4 +- 4 files changed, 137 insertions(+), 69 deletions(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index a88f673e0b..2ec6a7b0ce 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -65,6 +65,8 @@ typedef struct SSysTableScanInfo { SSDataBlock* pRes; int64_t numOfBlocks; // extract basic running information. SLoadRemoteDataInfo loadInfo; + + int32_t tbnameSlotId; } SSysTableScanInfo; typedef struct { @@ -346,6 +348,11 @@ static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result); static int32_t optSysCheckOper(SNode* pOpear); static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt); +static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableScanInfo* pInfo, const char* name, + SExecTaskInfo* pTaskInfo); +void extractTbnameSlotId(SSysTableScanInfo* pInfo, const SScanPhysiNode* pScanNode); +static SSDataBlock* sysTableScanHandleFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, + const char* name, SSDataBlock* pBlock); __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { *reverse = true; @@ -1309,83 +1316,111 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { getDBNameFromCondition(pInfo->pCondition, dbName); sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); } - + SSDataBlock* pBlock = NULL; if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { - return sysTableScanUserTables(pOperator); + pBlock = sysTableScanUserTables(pOperator); } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { - return sysTableScanUserTags(pOperator); + pBlock = sysTableScanUserTags(pOperator); } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite && IS_SYS_DBNAME(dbName)) { - return sysTableScanUserSTables(pOperator); + pBlock = sysTableScanUserSTables(pOperator); } else { // load the meta from mnode of the given epset - if (pOperator->status == OP_EXEC_DONE) { + pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo); + } + + return sysTableScanHandleFillTbName(pOperator, pInfo, name, pBlock); +} + +static SSDataBlock* sysTableScanHandleFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, + const char* name, SSDataBlock* pBlock) { + if (pBlock != NULL) { + if (pInfo->tbnameSlotId != -1) { + SColumnInfoData* pColumnInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, pInfo->tbnameSlotId); + char varTbName[TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE] = {0}; + memcpy(varDataVal(varTbName), name, strlen(name)); + varDataSetLen(varTbName, strlen(name)); + for (int i = 0; i < pBlock->info.rows; ++i) { + colDataAppend(pColumnInfoData, i, varTbName, NULL); + } + doFilterResult(pBlock, pOperator->exprSupp.pFilterInfo); + } + } + if (pBlock && pBlock->info.rows != 0) { + return pBlock; + } else { + return NULL; + } +} + +static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableScanInfo* pInfo, const char* name, + SExecTaskInfo* pTaskInfo) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + while (1) { + int64_t startTs = taosGetTimestampUs(); + tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); + + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); + char* buf1 = taosMemoryCalloc(1, contLen); + tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - while (1) { - int64_t startTs = taosGetTimestampUs(); - tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); - tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); + int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE + : TDMT_MND_SYSTABLE_RETRIEVE; - int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); - char* buf1 = taosMemoryCalloc(1, contLen); - tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); + pMsgSendInfo->param = pOperator; + pMsgSendInfo->msgInfo.pData = buf1; + pMsgSendInfo->msgInfo.len = contLen; + pMsgSendInfo->msgType = msgType; + pMsgSendInfo->fp = loadSysTableCallback; + pMsgSendInfo->requestId = pTaskInfo->id.queryId; - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + int64_t transporterId = 0; + int32_t code = + asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo); + tsem_wait(&pInfo->ready); + + if (pTaskInfo->code) { + qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), + pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code)); + return NULL; + } + + SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; + pInfo->req.showId = pRsp->handle; + + if (pRsp->numOfRows == 0 || pRsp->completed) { + pOperator->status = OP_EXEC_DONE; + qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), + pRsp->numOfRows, pInfo->loadInfo.totalRows); + + if (pRsp->numOfRows == 0) { + taosMemoryFree(pRsp); return NULL; } + } - int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE - : TDMT_MND_SYSTABLE_RETRIEVE; + char* pStart = pRsp->data; + extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart); + updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); - pMsgSendInfo->param = pOperator; - pMsgSendInfo->msgInfo.pData = buf1; - pMsgSendInfo->msgInfo.len = contLen; - pMsgSendInfo->msgType = msgType; - pMsgSendInfo->fp = loadSysTableCallback; - pMsgSendInfo->requestId = pTaskInfo->id.queryId; - - int64_t transporterId = 0; - int32_t code = - asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo); - tsem_wait(&pInfo->ready); - - if (pTaskInfo->code) { - qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), - pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code)); - return NULL; - } - - SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; - pInfo->req.showId = pRsp->handle; - - if (pRsp->numOfRows == 0 || pRsp->completed) { - pOperator->status = OP_EXEC_DONE; - qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), - pRsp->numOfRows, pInfo->loadInfo.totalRows); - - if (pRsp->numOfRows == 0) { - taosMemoryFree(pRsp); - return NULL; - } - } - - char* pStart = pRsp->data; - extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart); - updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); - - // todo log the filter info - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - taosMemoryFree(pRsp); - if (pInfo->pRes->info.rows > 0) { - return pInfo->pRes; - } else if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } + // todo log the filter info + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + taosMemoryFree(pRsp); + if (pInfo->pRes->info.rows > 0) { + return pInfo->pRes; + } else if (pOperator->status == OP_EXEC_DONE) { + return NULL; } } } @@ -1407,6 +1442,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan goto _error; } + extractTbnameSlotId(pInfo, pScanNode); + pInfo->accountId = pScanPhyNode->accountId; pInfo->pUser = taosMemoryStrDup((void*)pUser); pInfo->sysInfo = pScanPhyNode->sysInfo; @@ -1449,6 +1486,26 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan return NULL; } +void extractTbnameSlotId(SSysTableScanInfo* pInfo, const SScanPhysiNode* pScanNode) { + pInfo->tbnameSlotId = -1; + if (pScanNode->pScanPseudoCols != NULL) { + SNode* pNode = NULL; + FOREACH(pNode, pScanNode->pScanPseudoCols) { + STargetNode* pTargetNode = NULL; + if (nodeType(pNode) == QUERY_NODE_TARGET) { + pTargetNode = (STargetNode*)pNode; + SNode* expr = pTargetNode->pExpr; + if (nodeType(expr) == QUERY_NODE_FUNCTION) { + SFunctionNode* pFuncNode = (SFunctionNode*)expr; + if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) { + pInfo->tbnameSlotId = pTargetNode->slotId; + } + } + } + } + } +} + void destroySysScanOperator(void* param) { SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param; tsem_destroy(&pInfo->ready); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index e1687fc3a5..30dc092f53 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2380,6 +2380,9 @@ static bool tagScanOptShouldBeOptimized(SLogicNode* pNode) { if (pScan->hasNormalCols) { return false; } + if (pScan->tableType == TSDB_SYSTEM_TABLE) { + return false; + } if (NULL == pNode->pParent || QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) || 1 != LIST_LENGTH(pNode->pParent->pChildren)) { return false; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index f5782dc937..a7eac2c853 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -333,13 +333,23 @@ static bool stbSplHasPartTbname(SNodeList* pPartKeys) { return false; } -static bool stbSplIsPartTableAgg(SAggLogicNode* pAgg) { - if (NULL != pAgg->pGroupKeys) { - return stbSplHasPartTbname(pAgg->pGroupKeys); +static bool stbSplNotSystemScan(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + return SCAN_TYPE_SYSTEM_TABLE != ((SScanLogicNode*)pNode)->scanType; + } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + return stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); + } else { + return true; } +} + +static bool stbSplIsPartTableAgg(SAggLogicNode* pAgg) { if (1 != LIST_LENGTH(pAgg->node.pChildren)) { return false; } + if (NULL != pAgg->pGroupKeys) { + return stbSplHasPartTbname(pAgg->pGroupKeys) && stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)); + } return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))); } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index c6a4a97f6e..dacc696705 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -20,12 +20,10 @@ #include "tglobal.h" static void dumpQueryPlan(SQueryPlan* pPlan) { - if (!tsQueryPlannerTrace) { - return; - } char* pStr = NULL; nodesNodeToString((SNode*)pPlan, false, &pStr, NULL); planDebugL("QID:0x%" PRIx64 " Query Plan: %s", pPlan->queryId, pStr); + printf("QueryPlan\n%s\n\n", pStr); taosMemoryFree(pStr); } From 0bef5fdc380a21e8596ab4fb9e2e266dca824c5f Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 26 Dec 2022 14:42:36 +0800 Subject: [PATCH 2/4] fix: rename symbol --- source/libs/executor/src/sysscanoperator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 2ec6a7b0ce..ac32b54f56 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -351,7 +351,7 @@ static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt); static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableScanInfo* pInfo, const char* name, SExecTaskInfo* pTaskInfo); void extractTbnameSlotId(SSysTableScanInfo* pInfo, const SScanPhysiNode* pScanNode); -static SSDataBlock* sysTableScanHandleFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, +static SSDataBlock* sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, const char* name, SSDataBlock* pBlock); __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { @@ -1328,10 +1328,10 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo); } - return sysTableScanHandleFillTbName(pOperator, pInfo, name, pBlock); + return sysTableScanFillTbName(pOperator, pInfo, name, pBlock); } -static SSDataBlock* sysTableScanHandleFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, +static SSDataBlock* sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, const char* name, SSDataBlock* pBlock) { if (pBlock != NULL) { if (pInfo->tbnameSlotId != -1) { From fb4a6f508ca574818cfef4f8676ee42554c6da75 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 27 Dec 2022 09:54:57 +0800 Subject: [PATCH 3/4] fix: revert planner.c --- source/libs/planner/src/planner.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index dacc696705..c6a4a97f6e 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -20,10 +20,12 @@ #include "tglobal.h" static void dumpQueryPlan(SQueryPlan* pPlan) { + if (!tsQueryPlannerTrace) { + return; + } char* pStr = NULL; nodesNodeToString((SNode*)pPlan, false, &pStr, NULL); planDebugL("QID:0x%" PRIx64 " Query Plan: %s", pPlan->queryId, pStr); - printf("QueryPlan\n%s\n\n", pStr); taosMemoryFree(pStr); } From ceea74ee7aab0de0ad8af25a63935cf5297bb0e1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 27 Dec 2022 09:58:22 +0800 Subject: [PATCH 4/4] enhance: remove assert --- source/libs/function/src/tudf.c | 1 - source/libs/function/src/udfd.c | 1 - 2 files changed, 2 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 0b309bc8f5..abc75655cb 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1224,7 +1224,6 @@ int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode * if (uvTask->rspBuf.base != NULL) { SUdfResponse rsp = {0}; void *buf = decodeUdfResponse(uvTask->rspBuf.base, &rsp); - assert(uvTask->rspBuf.len == POINTER_DISTANCE(buf, uvTask->rspBuf.base)); task->errCode = rsp.code; switch (task->type) { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 40c75ce6ba..451f7bd0e8 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -400,7 +400,6 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->info.ahandle; - ASSERT(pMsg->info.ahandle != NULL); if (pEpSet) { if (!isEpsetEqual(&global.mgmtEp.epSet, pEpSet)) {