From d894e959ee9347f7a14e78f4451fc07e8e5dae7e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 14 Jul 2022 20:16:39 +0800 Subject: [PATCH 1/3] fix: client module memory leak problem --- source/client/src/clientImpl.c | 8 ++------ source/client/src/clientMain.c | 4 +++- source/libs/planner/src/planOptimizer.c | 11 ++++++++--- source/libs/planner/test/planBasicTest.cpp | 2 ++ 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d846cb93af..1122ba395e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -934,7 +934,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM int32_t code = 0; pRequest->body.execMode = pQuery->execMode; - + switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: asyncExecLocalCmd(pRequest, pQuery); @@ -1006,10 +1006,6 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM break; } - // if (!keepQuery) { - // qDestroyQuery(pQuery); - // } - if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } @@ -1479,7 +1475,7 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU tsem_wait(&pParam->sem); } - if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) { + if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) { return NULL; } else { if (setupOneRowPtr) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 14a431feab..6ab16b722e 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -688,8 +688,10 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); launchAsyncQuery(pRequest, pQuery, pResultMeta); + qDestroyQuery(pQuery); } else { destorySqlParseWrapper(pWrapper); + qDestroyQuery(pQuery); if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); @@ -867,7 +869,7 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } else { pResultInfo->numOfRows = 0; } - + pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows); return; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 7915f4a138..a843eb80a9 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -124,9 +124,11 @@ static bool scanPathOptMayBeOptimized(SLogicNode* pNode) { QUERY_NODE_LOGIC_PLAN_PARTITION != nodeType(pNode->pParent))) { return false; } - if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) && WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType) || + if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) && + WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType) || (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode->pParent) && pNode->pParent->pParent && - QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent) && WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType)) { + QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent) && + WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType)) { return true; } if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode->pParent)) { @@ -2010,7 +2012,10 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic SFunctionNode* pFunc = (SFunctionNode*)pNode; int32_t len = snprintf(pFunc->functionName, sizeof(pFunc->functionName), "_cache_last_row"); pFunc->functionName[len] = '\0'; - fmGetFuncInfo(pFunc, NULL, 0); + int32_t code = fmGetFuncInfo(pFunc, NULL, 0); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } pAgg->hasLastRow = false; diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 8482698cd3..0cff136349 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -106,6 +106,8 @@ TEST_F(PlanBasicTest, lastRowFunc) { run("SELECT LAST_ROW(c1, c2) FROM t1"); + run("SELECT LAST_ROW(c1), c2 FROM t1"); + run("SELECT LAST_ROW(c1) FROM st1"); run("SELECT LAST_ROW(c1) FROM st1 PARTITION BY TBNAME"); From 31e29dfc381e34d45b8422f5d37011f6a1922e7c Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 15 Jul 2022 08:43:40 +0800 Subject: [PATCH 2/3] feat: the query uses the 'cachelast' option to decide whether to optimize the last_row function --- include/libs/nodes/plannodes.h | 1 + include/libs/nodes/querynodes.h | 1 + source/libs/parser/src/parAstParser.c | 3 ++ source/libs/parser/src/parTranslater.c | 32 ++++++++++++++++------ source/libs/planner/src/planLogicCreater.c | 1 + source/libs/planner/src/planOptimizer.c | 3 +- source/libs/planner/src/planPhysiCreater.c | 2 ++ 7 files changed, 33 insertions(+), 10 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index b807aeca22..280e9edf33 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -77,6 +77,7 @@ typedef struct SScanLogicNode { SArray* pSmaIndexes; SNodeList* pGroupTags; bool groupSort; + int8_t cacheLastMode; } SScanLogicNode; typedef struct SJoinLogicNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 234b554526..d58d5d4b8b 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -152,6 +152,7 @@ typedef struct SRealTableNode { char qualDbName[TSDB_DB_NAME_LEN]; // SHOW qualDbName.TABLES double ratio; SArray* pSmaIndexes; + int8_t cacheLastMode; } SRealTableNode; typedef struct STempTableNode { diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 2060c3da4c..3fba404eaa 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -140,6 +140,9 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, SRealTa if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_DNODE_VARIABLES))) { code = reserveDnodeRequiredInCache(pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code) { + code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pCxt->pMetaCache); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d108e86df1..15763c13a6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1772,6 +1772,15 @@ static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNo return TSDB_CODE_SUCCESS; } +static int32_t setTableCacheLastMode(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + SDbCfgInfo dbCfg = {0}; + int32_t code = getDBCfg(pCxt, pRealTable->table.dbName, &dbCfg); + if (TSDB_CODE_SUCCESS == code) { + pRealTable->cacheLastMode = dbCfg.cacheLast; + } + return code; +} + static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pTable)) { @@ -1791,6 +1800,9 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { if (TSDB_CODE_SUCCESS == code) { code = setTableIndex(pCxt, &name, pRealTable); } + if (TSDB_CODE_SUCCESS == code) { + code = setTableCacheLastMode(pCxt, &name, pRealTable); + } } pRealTable->table.precision = pRealTable->pMeta->tableInfo.precision; pRealTable->table.singleTable = isSingleTable(pRealTable); @@ -5413,11 +5425,12 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { } static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt, - const STag* pTag, uint64_t suid, const char* sTableNmae, SVgroupInfo* pVgInfo, SArray* tagName) { -// char dbFName[TSDB_DB_FNAME_LEN] = {0}; -// SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; -// strcpy(name.dbname, pStmt->dbName); -// tNameGetFullDbName(&name, dbFName); + const STag* pTag, uint64_t suid, const char* sTableNmae, SVgroupInfo* pVgInfo, + SArray* tagName) { + // char dbFName[TSDB_DB_FNAME_LEN] = {0}; + // SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; + // strcpy(name.dbname, pStmt->dbName); + // tNameGetFullDbName(&name, dbFName); struct SVCreateTbReq req = {0}; req.type = TD_CHILD_TABLE; @@ -5524,7 +5537,7 @@ static int32_t buildNormalTagVal(STranslateContext* pCxt, SSchema* pTagSchema, S if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { void* nodeVal = nodesGetValueFromNode(pVal); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; -// strcpy(val.colName, pTagSchema->name); + // strcpy(val.colName, pTagSchema->name); if (IS_VAR_DATA_TYPE(pTagSchema->type)) { val.pData = varDataVal(nodeVal); val.nData = varDataLen(nodeVal); @@ -5621,7 +5634,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) { char* tmpVal = nodesGetValueFromNode(pVal); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; -// strcpy(val.colName, pTagSchema->name); + // strcpy(val.colName, pTagSchema->name); if (IS_VAR_DATA_TYPE(pTagSchema->type)) { val.pData = varDataVal(tmpVal); val.nData = varDataLen(tmpVal); @@ -5664,7 +5677,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta); } - STag* pTag = NULL; + STag* pTag = NULL; SArray* tagName = taosArrayInit(8, TSDB_COL_NAME_LEN); if (TSDB_CODE_SUCCESS == code) { @@ -5680,7 +5693,8 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &info); } if (TSDB_CODE_SUCCESS == code) { - addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt, pTag, pSuperTableMeta->uid, pStmt->useTableName, &info, tagName); + addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt, pTag, pSuperTableMeta->uid, + pStmt->useTableName, &info, tagName); } taosArrayDestroy(tagName); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 190ec3a1f6..4a062a86db 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -244,6 +244,7 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT pScan->showRewrite = pCxt->pPlanCxt->showRewrite; pScan->ratio = pRealTable->ratio; pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; + pScan->cacheLastMode = pRealTable->cacheLastMode; *pLogicNode = (SLogicNode*)pScan; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index a843eb80a9..1eb807dfbe 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1985,7 +1985,8 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || !(((SAggLogicNode*)pNode)->hasLastRow) || NULL != ((SAggLogicNode*)pNode)->pGroupKeys || 1 != LIST_LENGTH(pNode->pChildren) || QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0)) || - NULL != ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->node.pConditions) { + NULL != ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->node.pConditions || + 0 == ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->cacheLastMode) { return false; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index c4cc31c535..2736ff7e08 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -515,6 +515,8 @@ static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSu } pScan->groupSort = pScanLogicNode->groupSort; + vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } From 979b20189d94b092df18097916e7f4b5e1549610 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 15 Jul 2022 09:07:19 +0800 Subject: [PATCH 3/3] feat: the query uses the 'cachelast' option to decide whether to optimize the last_row function --- source/libs/parser/src/parTranslater.c | 4 ++++ source/libs/parser/test/mockCatalog.cpp | 2 ++ 2 files changed, 6 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 15763c13a6..0ada99ce68 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1773,6 +1773,10 @@ static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNo } static int32_t setTableCacheLastMode(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) { + return TSDB_CODE_SUCCESS; + } + SDbCfgInfo dbCfg = {0}; int32_t code = getDBCfg(pCxt, pRealTable->table.dbName, &dbCfg); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 6eafa0555b..d054a3434e 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -234,6 +234,8 @@ void generateDnodes(MockCatalogService* mcs) { } void generateDatabases(MockCatalogService* mcs) { + mcs->createDatabase(TSDB_INFORMATION_SCHEMA_DB); + mcs->createDatabase(TSDB_PERFORMANCE_SCHEMA_DB); mcs->createDatabase("test"); mcs->createDatabase("rollup_db", true); }