diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index ca7879fe41..7df13846c2 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -110,8 +110,8 @@ typedef struct SScanLogicNode { int8_t igCheckUpdate; SArray* pSmaIndexes; SArray* pTsmas; - SArray* pTsmaTargetCTbVgInfo; - SArray* pTsmaTargetCTbInfo; + SArray* pTsmaTargetTbVgInfo; + SArray* pTsmaTargetTbInfo; SNodeList* pGroupTags; bool groupSort; SNodeList* pTags; // for create stream diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 3f3d4cf031..602bfdde39 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -191,9 +191,9 @@ typedef struct STableNode { struct STableMeta; typedef struct STsmaTargetCTbInfo { - char ctableName[TSDB_TABLE_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; // child table or normal table name uint64_t uid; -} STsmaTargetCTbInfo; +} STsmaTargetTbInfo; typedef struct SRealTableNode { STableNode table; // QUERY_NODE_REAL_TABLE @@ -204,8 +204,8 @@ typedef struct SRealTableNode { SArray* pSmaIndexes; int8_t cacheLastMode; SArray* pTsmas; - SArray* tsmaTargetCTbVgInfo; // SArray - SArray* tsmaTargetCTbInfo; // SArray + SArray* tsmaTargetTbVgInfo; // SArray, used for child table or normal table only + SArray* tsmaTargetTbInfo; // SArray, used for child table or normal table only } SRealTableNode; typedef struct STempTableNode { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index d032bf241f..2f080c2b81 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -3259,7 +3259,7 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx // get tb cache pName = taosArrayGet(pList, i); pTbCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); - if (!pTbCache) { + if (!pTbCache || !pTbCache->pMeta) { ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname); ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL); taosArrayPush(pCtx->pResList, &(SMetaRes){0}); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 1ee3216b7b..4c25f60294 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -483,6 +483,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { CLONE_OBJECT_FIELD(pFuncTypes, functParamClone); COPY_SCALAR_FIELD(paraTablesSort); COPY_SCALAR_FIELD(smallDataTsSort); + COPY_SCALAR_FIELD(needSplit); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index e7d5313b4c..9106a39cb8 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -816,8 +816,8 @@ void nodesDestroyNode(SNode* pNode) { taosMemoryFreeClear(pReal->pMeta); taosMemoryFreeClear(pReal->pVgroupList); taosArrayDestroyEx(pReal->pSmaIndexes, destroySmaIndex); - taosArrayDestroyP(pReal->tsmaTargetCTbVgInfo, taosMemoryFree); - taosArrayDestroy(pReal->tsmaTargetCTbInfo); + taosArrayDestroyP(pReal->tsmaTargetTbVgInfo, taosMemoryFree); + taosArrayDestroy(pReal->tsmaTargetTbInfo); break; } case QUERY_NODE_TEMP_TABLE: @@ -1310,8 +1310,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyList(pLogicNode->pTags); nodesDestroyNode(pLogicNode->pSubtable); taosArrayDestroyEx(pLogicNode->pFuncTypes, destroyFuncParam); - taosArrayDestroyP(pLogicNode->pTsmaTargetCTbVgInfo, taosMemoryFree); - taosArrayDestroy(pLogicNode->pTsmaTargetCTbInfo); + taosArrayDestroyP(pLogicNode->pTsmaTargetTbVgInfo, taosMemoryFree); + taosArrayDestroy(pLogicNode->pTsmaTargetTbInfo); break; } case QUERY_NODE_LOGIC_PLAN_JOIN: { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index e773524bd4..fdea1f45d5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3663,26 +3663,27 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo if (isSelectStmt(pCxt->pCurrStmt) && pRealTable->pMeta->tableType != TSDB_SYSTEM_TABLE) { code = getTableTsmas(pCxt, pName, &pRealTable->pTsmas); // if select from a child table, fetch it's corresponding tsma target child table infos - if (TSDB_CODE_SUCCESS == code && pRealTable->pTsmas && pRealTable->pMeta->tableType == TSDB_CHILD_TABLE) { - if (pRealTable->tsmaTargetCTbVgInfo) { - taosArrayDestroyP(pRealTable->tsmaTargetCTbVgInfo, taosMemoryFree); - pRealTable->tsmaTargetCTbVgInfo = NULL; + if (TSDB_CODE_SUCCESS == code && pRealTable->pTsmas && + (pRealTable->pMeta->tableType == TSDB_CHILD_TABLE || pRealTable->pMeta->tableType == TSDB_NORMAL_TABLE)) { + if (pRealTable->tsmaTargetTbVgInfo) { + taosArrayDestroyP(pRealTable->tsmaTargetTbVgInfo, taosMemoryFree); + pRealTable->tsmaTargetTbVgInfo = NULL; } for (int32_t i = 0; i < pRealTable->pTsmas->size; ++i) { STableTSMAInfo* pTsma = taosArrayGetP(pRealTable->pTsmas, i); - SName tsmaTargetCTbName = {0}; - toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetCTbName); - int32_t len = snprintf(tsmaTargetCTbName.tname, TSDB_TABLE_NAME_LEN, "%s.%s", pTsma->dbFName, pTsma->name); - len = taosCreateMD5Hash(tsmaTargetCTbName.tname, len); - sprintf(tsmaTargetCTbName.tname + len, "_%s", pRealTable->table.tableName); + SName tsmaTargetTbName = {0}; + toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName); + int32_t len = snprintf(tsmaTargetTbName.tname, TSDB_TABLE_NAME_LEN, "%s.%s", pTsma->dbFName, pTsma->name); + len = taosCreateMD5Hash(tsmaTargetTbName.tname, len); + sprintf(tsmaTargetTbName.tname + len, "_%s", pRealTable->table.tableName); SVgroupInfo vgInfo = {0}; bool exists = false; - code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &tsmaTargetCTbName, &vgInfo, &exists); + code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &tsmaTargetTbName, &vgInfo, &exists); if (TSDB_CODE_SUCCESS == code) { ASSERT(exists); - if (!pRealTable->tsmaTargetCTbVgInfo) { - pRealTable->tsmaTargetCTbVgInfo = taosArrayInit(pRealTable->pTsmas->size, POINTER_BYTES); - if (!pRealTable->tsmaTargetCTbVgInfo) { + if (!pRealTable->tsmaTargetTbVgInfo) { + pRealTable->tsmaTargetTbVgInfo = taosArrayInit(pRealTable->pTsmas->size, POINTER_BYTES); + if (!pRealTable->tsmaTargetTbVgInfo) { code = TSDB_CODE_OUT_OF_MEMORY; break; } @@ -3694,7 +3695,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo } pVgpsInfo->numOfVgroups = 1; pVgpsInfo->vgroups[0] = vgInfo; - taosArrayPush(pRealTable->tsmaTargetCTbVgInfo, &pVgpsInfo); + taosArrayPush(pRealTable->tsmaTargetTbVgInfo, &pVgpsInfo); } else { break; } @@ -3705,23 +3706,25 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo .requestId = pCxt->pParseCxt->requestId, .requestObjRefId = pCxt->pParseCxt->requestRid, .mgmtEps = pCxt->pParseCxt->mgmtEpSet}; - code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, &conn, &tsmaTargetCTbName, &pTableMeta); + code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, &conn, &tsmaTargetTbName, &pTableMeta); + } + STsmaTargetTbInfo ctbInfo = {0}; + if (!pRealTable->tsmaTargetTbInfo) { + pRealTable->tsmaTargetTbInfo = taosArrayInit(pRealTable->pTsmas->size, sizeof(STsmaTargetTbInfo)); + if (!pRealTable->tsmaTargetTbInfo) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } } if (code == TSDB_CODE_SUCCESS) { - STsmaTargetCTbInfo ctbInfo = {0}; - sprintf(ctbInfo.ctableName, "%s", tsmaTargetCTbName.tname); + sprintf(ctbInfo.tableName, "%s", tsmaTargetTbName.tname); ctbInfo.uid = pTableMeta->uid; taosMemoryFree(pTableMeta); - if (!pRealTable->tsmaTargetCTbInfo) { - pRealTable->tsmaTargetCTbInfo = taosArrayInit(pRealTable->pTsmas->size, sizeof(STsmaTargetCTbInfo)); - if (!pRealTable->tsmaTargetCTbInfo) { - code = TSDB_CODE_OUT_OF_MEMORY; - break; - } - } - - taosArrayPush(pRealTable->tsmaTargetCTbInfo, &ctbInfo); + } else if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { + // ignore table not exists error + code = TSDB_CODE_SUCCESS; } + taosArrayPush(pRealTable->tsmaTargetTbInfo, &ctbInfo); } } } @@ -5681,8 +5684,8 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* vgsInfo = NULL; if (pInfo->pRealTable->pTsmas) { - pInfo->pRealTable->tsmaTargetCTbVgInfo = taosArrayInit(pInfo->pRealTable->pTsmas->size, POINTER_BYTES); - if (!pInfo->pRealTable->tsmaTargetCTbVgInfo) return TSDB_CODE_OUT_OF_MEMORY; + pInfo->pRealTable->tsmaTargetTbVgInfo = taosArrayInit(pInfo->pRealTable->pTsmas->size, POINTER_BYTES); + if (!pInfo->pRealTable->tsmaTargetTbVgInfo) return TSDB_CODE_OUT_OF_MEMORY; for (int32_t i = 0; i < pInfo->pRealTable->pTsmas->size; ++i) { STableTSMAInfo* pTsma = taosArrayGetP(pInfo->pRealTable->pTsmas, i); @@ -5708,7 +5711,7 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* if (TSDB_CODE_SUCCESS == code) { findVgroupsFromEqualTbname(pCxt, pTbNames, pInfo->pRealTable->table.dbName, numOfVgs, vgsInfo); if (vgsInfo->numOfVgroups != 0) { - taosArrayPush(pInfo->pRealTable->tsmaTargetCTbVgInfo, &vgsInfo); + taosArrayPush(pInfo->pRealTable->tsmaTargetTbVgInfo, &vgsInfo); } else { taosMemoryFree(vgsInfo); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 9e9d0ec805..cceabcbf50 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -365,8 +365,8 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT TSWAP(pScan->pVgroupList, pRealTable->pVgroupList); TSWAP(pScan->pSmaIndexes, pRealTable->pSmaIndexes); TSWAP(pScan->pTsmas, pRealTable->pTsmas); - TSWAP(pScan->pTsmaTargetCTbVgInfo, pRealTable->tsmaTargetCTbVgInfo); - TSWAP(pScan->pTsmaTargetCTbInfo, pRealTable->tsmaTargetCTbInfo); + TSWAP(pScan->pTsmaTargetTbVgInfo, pRealTable->tsmaTargetTbVgInfo); + TSWAP(pScan->pTsmaTargetTbInfo, pRealTable->tsmaTargetTbInfo); pScan->tableId = pRealTable->pMeta->uid; pScan->stableId = pRealTable->pMeta->suid; pScan->tableType = pRealTable->pMeta->tableType; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 5d58e1b64e..2844a979bc 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -6287,7 +6287,7 @@ static int32_t tsmaOptRewriteScan(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pNew pNewScan->pScanCols = tsmaOptCreateTsmaScanCols(pTsma, pTsmaOptCtx->pAggFuncs); if (!pNewScan->pScanCols) code = TSDB_CODE_OUT_OF_MEMORY; } - if (pNewScan->tableType == TSDB_CHILD_TABLE) { + if (pNewScan->tableType == TSDB_CHILD_TABLE) { // TODO remvoe it } if (code == TSDB_CODE_SUCCESS && pPkTsCol) { tstrncpy(pPkTsCol->tableName, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); @@ -6310,11 +6310,11 @@ static int32_t tsmaOptRewriteScan(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pNew if (code == TSDB_CODE_SUCCESS) { code = tsmaOptRewriteNodeList(pNewScan->pGroupTags, pTsmaOptCtx, pTsma, true, true); } - if (pNewScan->pTsmaTargetCTbVgInfo && pNewScan->pTsmaTargetCTbVgInfo->size > 0) { - for (int32_t i = 0; i < taosArrayGetSize(pNewScan->pTsmas); ++i) { - STableTSMAInfo* pTsmaInfo = taosArrayGetP(pNewScan->pTsmas, i); + if (pTsmaOptCtx->pScan->pTsmaTargetTbVgInfo && pTsmaOptCtx->pScan->pTsmaTargetTbVgInfo->size > 0) { + for (int32_t i = 0; i < taosArrayGetSize(pTsmaOptCtx->pScan->pTsmas); ++i) { + STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmaOptCtx->pScan->pTsmas, i); if (pTsmaInfo == pTsma->pTsma) { - SVgroupsInfo* pVgpsInfo = taosArrayGetP(pNewScan->pTsmaTargetCTbVgInfo, i); + const SVgroupsInfo* pVgpsInfo = taosArrayGetP(pTsmaOptCtx->pScan->pTsmaTargetTbVgInfo, i); taosMemoryFreeClear(pNewScan->pVgroupList); int32_t len = sizeof(int32_t) + sizeof(SVgroupInfo) * pVgpsInfo->numOfVgroups; pNewScan->pVgroupList = taosMemoryCalloc(1, len); @@ -6533,19 +6533,19 @@ static int32_t tsmaOptGeneratePlan(STSMAOptCtx* pTsmaOptCtx) { // TODO if no used tsmas skip generating plans for (int32_t i = 0; i < pTsmaOptCtx->pUsedTsmas->size; ++i) { STSMAOptUsefulTsma* pTsma = taosArrayGet(pTsmaOptCtx->pUsedTsmas, i); - if (pTsma->pTsma) { - if (pTsmaOptCtx->pScan->tableType == TSDB_CHILD_TABLE) { - for (int32_t j = 0; j < pTsmaOptCtx->pScan->pTsmas->size; ++j) { - if (taosArrayGetP(pTsmaOptCtx->pScan->pTsmas, j) == pTsma->pTsma) { - const STsmaTargetCTbInfo* pCtbInfo = taosArrayGet(pTsmaOptCtx->pScan->pTsmaTargetCTbInfo, j); - strcpy(pTsma->targetTbName, pCtbInfo->ctableName); - pTsma->targetTbUid = pCtbInfo->uid; - } + if (!pTsma->pTsma) continue; + if (pTsmaOptCtx->pScan->tableType == TSDB_CHILD_TABLE || pTsmaOptCtx->pScan->tableType == TSDB_NORMAL_TABLE) { + for (int32_t j = 0; j < pTsmaOptCtx->pScan->pTsmas->size; ++j) { + if (taosArrayGetP(pTsmaOptCtx->pScan->pTsmas, j) == pTsma->pTsma) { + const STsmaTargetTbInfo* ptbInfo = taosArrayGet(pTsmaOptCtx->pScan->pTsmaTargetTbInfo, j); + ASSERT(ptbInfo->uid != 0); + strcpy(pTsma->targetTbName, ptbInfo->tableName); + pTsma->targetTbUid = ptbInfo->uid; } - } else { - strcpy(pTsma->targetTbName, pTsma->pTsma->targetTb); - pTsma->targetTbUid = pTsma->pTsma->destTbUid; } + } else { + strcpy(pTsma->targetTbName, pTsma->pTsma->targetTb); + pTsma->targetTbUid = pTsma->pTsma->destTbUid; } } diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index b0a1ff1b2a..5829d91dce 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -339,7 +339,8 @@ class TSMATestSQLGenerator: else: return auto_order_by[:-1] - def generate_one(self, select_list: str, tb: str, order_by_list: str, interval_list: List[str] = []) -> str: + def generate_one(self, select_list: str, possible_tbs: List, order_by_list: str, interval_list: List[str] = []) -> str: + tb = random.choice(possible_tbs) where = self.generate_where() interval = self.generate_interval(interval_list) (partition_by, partition_by_list) = self.generate_partition_by() @@ -492,7 +493,7 @@ class TSMATestSQLGenerator: class TDTestCase: updatecfgDict = {'debugFlag': 143, 'asynclog': 0} def __init__(self): - self.vgroups = 1 + self.vgroups = 4 self.ctbNum = 10 self.rowsPerTbl = 10000 self.duraion = '1h' @@ -610,6 +611,8 @@ class TDTestCase: if ctx.has_tsma(): if ctx.used_tsmas[0].name == tsma_name + UsedTsma.TSMA_RES_STB_POSTFIX: break + elif ctx.used_tsmas[0].name.find('_') == 32 and ctx.used_tsmas[0].name[33:] == tb: + break else: time.sleep(1) else: @@ -720,12 +723,17 @@ class TDTestCase: interval_list = ['1s', '5s', '60s', '1m', '10m', '20m', '30m', '59s', '1h', '120s', '1200', '2h', '90m', '1d'] opts: TSMATesterSQLGeneratorOptions = TSMATesterSQLGeneratorOptions() - opts.partition_by = True opts.interval = True opts.where_ts_range = True for _ in range(1, 100): + opts.partition_by = True sql_generator = TSMATestSQLGenerator(opts) - sql = sql_generator.generate_one('avg(c1), avg(c2)', 'meters', '', interval_list) + sql = sql_generator.generate_one('avg(c1), avg(c2)', ['meters', 't1', 't9'], '', interval_list) + ctxs.append(TSMAQCBuilder().with_sql(sql).ignore_query_table().ignore_res_order(sql_generator.can_ignore_res_order()).get_qc()) + + opts.partition_by = False + sql_generator = TSMATestSQLGenerator(opts) + sql = sql_generator.generate_one('avg(c1), avg(c2)', ['norm_tb', 't5'], '', interval_list) ctxs.append(TSMAQCBuilder().with_sql(sql).ignore_query_table().ignore_res_order(sql_generator.can_ignore_res_order()).get_qc()) return ctxs