diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index e4e50f69ee..ca7879fe41 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -127,6 +127,7 @@ typedef struct SScanLogicNode { SArray* pFuncTypes; // for last, last_row bool paraTablesSort; // for table merge scan bool smallDataTsSort; // disable row id sort for table merge scan + bool needSplit; } SScanLogicNode; typedef struct SJoinLogicNode { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8a37a22d29..5d58e1b64e 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -6310,7 +6310,7 @@ static int32_t tsmaOptRewriteScan(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pNew if (code == TSDB_CODE_SUCCESS) { code = tsmaOptRewriteNodeList(pNewScan->pGroupTags, pTsmaOptCtx, pTsma, true, true); } - if (pNewScan->pTsmaTargetCTbVgInfo) { + if (pNewScan->pTsmaTargetCTbVgInfo && pNewScan->pTsmaTargetCTbVgInfo->size > 0) { for (int32_t i = 0; i < taosArrayGetSize(pNewScan->pTsmas); ++i) { STableTSMAInfo* pTsmaInfo = taosArrayGetP(pNewScan->pTsmas, i); if (pTsmaInfo == pTsma->pTsma) { @@ -6528,6 +6528,7 @@ static int32_t tsmaOptGeneratePlan(STSMAOptCtx* pTsmaOptCtx) { int32_t code = 0; const STSMAOptUsefulTsma* pTsma = NULL; SNodeList* pAggFuncs = NULL; + bool hasSubPlan = false; // TODO if no used tsmas skip generating plans for (int32_t i = 0; i < pTsmaOptCtx->pUsedTsmas->size; ++i) { @@ -6557,6 +6558,7 @@ static int32_t tsmaOptGeneratePlan(STSMAOptCtx* pTsmaOptCtx) { } pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pTsmaOptCtx->generatedSubPlans[i - 1] = pSubplan; + hasSubPlan = true; SLogicNode* pParent = (SLogicNode*)nodesCloneNode((SNode*)pTsmaOptCtx->pParent); if (!pParent) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -6574,6 +6576,7 @@ static int32_t tsmaOptGeneratePlan(STSMAOptCtx* pTsmaOptCtx) { if (code == TSDB_CODE_SUCCESS) { pTsma = taosArrayGet(pTsmaOptCtx->pUsedTsmas, 0); + pTsmaOptCtx->pScan->needSplit = hasSubPlan; code = tsmaOptRewriteScan(pTsmaOptCtx, pTsmaOptCtx->pScan, pTsma); if (code == TSDB_CODE_SUCCESS && pTsma->pTsma) { code = tsmaOptRevisePlan2(pTsmaOptCtx, pTsmaOptCtx->pParent, pTsmaOptCtx->pScan, pTsma); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 6f2c07683c..bbd54ab846 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -39,6 +39,11 @@ typedef struct SSplitRule { FSplit splitFunc; } SSplitRule; +typedef struct SFindSplitNodeCtx { + const SSplitContext* pSplitCtx; + const SLogicSubplan* pSubplan; +} SFindSplitNodeCtx; + typedef bool (*FSplFindSplitNode)(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode, void* pInfo); static int32_t stbSplCreateMergeKeys(SNodeList* pSortKeys, SNodeList* pTargets, SNodeList** pOutput); @@ -232,7 +237,7 @@ static bool stbSplHasGatherExecFunc(const SNodeList* pFuncs) { } static bool stbSplIsMultiTbScan(bool streamQuery, SScanLogicNode* pScan) { - return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1); + return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1) || pScan->needSplit; } static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { @@ -249,6 +254,13 @@ static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild)) { return true; } + + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild)) { + if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) || (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode) && + ((SWindowLogicNode*)pNode)->winType == WINDOW_TYPE_INTERVAL)) { + return ((SScanLogicNode*)pChild)->needSplit; + } + } return false; } @@ -312,7 +324,8 @@ static bool stbSplIsTableCountQuery(SLogicNode* pNode) { return QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && SCAN_TYPE_TABLE_COUNT == ((SScanLogicNode*)pChild)->scanType; } -static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { +static bool stbSplNeedSplit(SFindSplitNodeCtx* pCtx, SLogicNode* pNode) { + bool streamQuery = pCtx->pSplitCtx->pPlanCxt->streamQuery; switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: return streamQuery ? false : stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pNode); @@ -336,7 +349,8 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { static bool stbSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode, SStableSplitInfo* pInfo) { - if (stbSplNeedSplit(pCxt->pPlanCxt->streamQuery, pNode)) { + SFindSplitNodeCtx ctx = {.pSplitCtx = pCxt, .pSubplan = pSubplan}; + if (stbSplNeedSplit(&ctx, pNode)) { pInfo->pSplitNode = pNode; pInfo->pSubplan = pSubplan; return true; diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index 12f9f353d5..b0a1ff1b2a 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -243,6 +243,7 @@ class TSMATesterSQLGeneratorOptions: self.column_num: int = 9 ### c1 - c10 self.tags_prefix: str = 't' self.tag_num: int = 6 ### t1 - t6 + self.char_tag_idx: List = [2,3] self.child_table_name_prefix: str = 't' self.child_table_num: int = 10 #### t0 - t9 self.interval: bool = False @@ -292,7 +293,7 @@ class TSMATestSQLGenerator: substr = f'SUBSTR({name}, {start}, {len})' lower = f'LOWER({name})' ltrim = f'LTRIM({name})' - return [concat1, concat2, concat3, substr, lower, ltrim, name] + return [concat1, concat2, concat3, substr, substr, lower, lower, ltrim, name] def generate_depthed_str_func(self, name: str, depth: int) -> str: if depth == 1: @@ -307,6 +308,9 @@ class TSMATestSQLGenerator: ret = self.generate_depthed_str_func(column_name, depth) tdLog.debug(f'generating str func: {ret}') return ret + + def generate_integer_func(self, column_name: str, depth: int = 0) -> str: + pass def get_random_type(self, funcs): rand: int = randrange(1, len(funcs)) @@ -420,7 +424,7 @@ class TSMATestSQLGenerator: for _ in range(used_tag_num): tag_idx = random.randint(1,self.opts_.tag_num) tag_name = self.opts_.tags_prefix + f'{tag_idx}' - if False and random.random() < 0.5: + if random.random() < 0.5 and tag_idx in self.opts_.char_tag_idx: tag_func = self.generate_str_func(tag_name, 2) else: tag_func = tag_name @@ -450,7 +454,7 @@ class TSMATestSQLGenerator: ret = '' rand = random.random() if rand < 0.4: - if False and random.random() < 0.5: + if random.random() < 0.5: ret = self.generate_str_func('tbname', 3) else: ret = 'tbname' @@ -488,7 +492,7 @@ class TSMATestSQLGenerator: class TDTestCase: updatecfgDict = {'debugFlag': 143, 'asynclog': 0} def __init__(self): - self.vgroups = 4 + self.vgroups = 1 self.ctbNum = 10 self.rowsPerTbl = 10000 self.duraion = '1h' @@ -719,7 +723,7 @@ class TDTestCase: opts.partition_by = True opts.interval = True opts.where_ts_range = True - for _ in range(1, 10000): + for _ in range(1, 100): sql_generator = TSMATestSQLGenerator(opts) sql = sql_generator.generate_one('avg(c1), avg(c2)', 'meters', '', interval_list) ctxs.append(TSMAQCBuilder().with_sql(sql).ignore_query_table().ignore_res_order(sql_generator.can_ignore_res_order()).get_qc()) @@ -832,6 +836,8 @@ class TDTestCase: sql = 'select avg(c1) + 1, tbname from meters group by tbname having avg(c1) > 0 and tbname = "t1" order by tbname' ctxs.append(TSMAQCBuilder().with_sql(sql).should_query_with_tsma('tsma2').get_qc()) + sql = 'select avg(c1) + 1, tbname from meters group by tbname having avg(c1) > 0 and tbname like "t%" order by tbname' + ctxs.append(TSMAQCBuilder().with_sql(sql).should_query_with_tsma('tsma2').get_qc()) return ctxs