xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix norm_tb child tb tsma query

This commit is contained in:
wangjiaming0909 2024-03-06 14:50:16 +08:00
parent 79b726fdbc
commit c9be37bdc6
9 changed files with 74 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/libs/nodes/plannodes.h
View File

@ -110,8 +110,8 @@ typedef struct SScanLogicNode {
int8_t igCheckUpdate;
SArray* pSmaIndexes;
SArray* pTsmas;
SArray* pTsmaTargetCTbVgInfo;
SArray* pTsmaTargetCTbInfo;
SArray* pTsmaTargetTbVgInfo;
SArray* pTsmaTargetTbInfo;
SNodeList* pGroupTags;
bool groupSort;
SNodeList* pTags; // for create stream

8
include/libs/nodes/querynodes.h
View File

@ -191,9 +191,9 @@ typedef struct STableNode {
struct STableMeta;
typedef struct STsmaTargetCTbInfo {
char ctableName[TSDB_TABLE_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; // child table or normal table name
uint64_t uid;
} STsmaTargetCTbInfo;
} STsmaTargetTbInfo;
typedef struct SRealTableNode {
STableNode table; // QUERY_NODE_REAL_TABLE
@ -204,8 +204,8 @@ typedef struct SRealTableNode {
SArray* pSmaIndexes;
int8_t cacheLastMode;
SArray* pTsmas;
SArray* tsmaTargetCTbVgInfo; // SArray<SVgroupsInfo*>
SArray* tsmaTargetCTbInfo; // SArray<STsmaTargetCTbInfo>
SArray* tsmaTargetTbVgInfo; // SArray<SVgroupsInfo*>, used for child table or normal table only
SArray* tsmaTargetTbInfo; // SArray<STsmaTargetTbInfo>, used for child table or normal table only
} SRealTableNode;
typedef struct STempTableNode {

2
source/libs/catalog/src/ctgCache.c
View File

@ -3259,7 +3259,7 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx
// get tb cache
pName = taosArrayGet(pList, i);
pTbCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname));
if (!pTbCache) {
if (!pTbCache || !pTbCache->pMeta) {
ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname);
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
taosArrayPush(pCtx->pResList, &(SMetaRes){0});

1
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -483,6 +483,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
CLONE_OBJECT_FIELD(pFuncTypes, functParamClone);
COPY_SCALAR_FIELD(paraTablesSort);
COPY_SCALAR_FIELD(smallDataTsSort);
COPY_SCALAR_FIELD(needSplit);
return TSDB_CODE_SUCCESS;
}

8
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -816,8 +816,8 @@ void nodesDestroyNode(SNode* pNode) {
taosMemoryFreeClear(pReal->pMeta);
taosMemoryFreeClear(pReal->pVgroupList);
taosArrayDestroyEx(pReal->pSmaIndexes, destroySmaIndex);
taosArrayDestroyP(pReal->tsmaTargetCTbVgInfo, taosMemoryFree);
taosArrayDestroy(pReal->tsmaTargetCTbInfo);
taosArrayDestroyP(pReal->tsmaTargetTbVgInfo, taosMemoryFree);
taosArrayDestroy(pReal->tsmaTargetTbInfo);
break;
}
case QUERY_NODE_TEMP_TABLE:
@ -1310,8 +1310,8 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyList(pLogicNode->pTags);
nodesDestroyNode(pLogicNode->pSubtable);
taosArrayDestroyEx(pLogicNode->pFuncTypes, destroyFuncParam);
taosArrayDestroyP(pLogicNode->pTsmaTargetCTbVgInfo, taosMemoryFree);
taosArrayDestroy(pLogicNode->pTsmaTargetCTbInfo);
taosArrayDestroyP(pLogicNode->pTsmaTargetTbVgInfo, taosMemoryFree);
taosArrayDestroy(pLogicNode->pTsmaTargetTbInfo);
break;
}
case QUERY_NODE_LOGIC_PLAN_JOIN: {

59
source/libs/parser/src/parTranslater.c
View File

@ -3663,26 +3663,27 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo
if (isSelectStmt(pCxt->pCurrStmt) && pRealTable->pMeta->tableType != TSDB_SYSTEM_TABLE) {
code = getTableTsmas(pCxt, pName, &pRealTable->pTsmas);
// if select from a child table, fetch it's corresponding tsma target child table infos
if (TSDB_CODE_SUCCESS == code && pRealTable->pTsmas && pRealTable->pMeta->tableType == TSDB_CHILD_TABLE) {
if (pRealTable->tsmaTargetCTbVgInfo) {
taosArrayDestroyP(pRealTable->tsmaTargetCTbVgInfo, taosMemoryFree);
pRealTable->tsmaTargetCTbVgInfo = NULL;
if (TSDB_CODE_SUCCESS == code && pRealTable->pTsmas &&
(pRealTable->pMeta->tableType == TSDB_CHILD_TABLE || pRealTable->pMeta->tableType == TSDB_NORMAL_TABLE)) {
if (pRealTable->tsmaTargetTbVgInfo) {
taosArrayDestroyP(pRealTable->tsmaTargetTbVgInfo, taosMemoryFree);
pRealTable->tsmaTargetTbVgInfo = NULL;
}
for (int32_t i = 0; i < pRealTable->pTsmas->size; ++i) {
STableTSMAInfo* pTsma = taosArrayGetP(pRealTable->pTsmas, i);
SName tsmaTargetCTbName = {0};
toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetCTbName);
int32_t len = snprintf(tsmaTargetCTbName.tname, TSDB_TABLE_NAME_LEN, "%s.%s", pTsma->dbFName, pTsma->name);
len = taosCreateMD5Hash(tsmaTargetCTbName.tname, len);
sprintf(tsmaTargetCTbName.tname + len, "_%s", pRealTable->table.tableName);
SName tsmaTargetTbName = {0};
toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName);
int32_t len = snprintf(tsmaTargetTbName.tname, TSDB_TABLE_NAME_LEN, "%s.%s", pTsma->dbFName, pTsma->name);
len = taosCreateMD5Hash(tsmaTargetTbName.tname, len);
sprintf(tsmaTargetTbName.tname + len, "_%s", pRealTable->table.tableName);
SVgroupInfo vgInfo = {0};
bool exists = false;
code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &tsmaTargetCTbName, &vgInfo, &exists);
code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &tsmaTargetTbName, &vgInfo, &exists);
if (TSDB_CODE_SUCCESS == code) {
ASSERT(exists);
if (!pRealTable->tsmaTargetCTbVgInfo) {
pRealTable->tsmaTargetCTbVgInfo = taosArrayInit(pRealTable->pTsmas->size, POINTER_BYTES);
if (!pRealTable->tsmaTargetCTbVgInfo) {
if (!pRealTable->tsmaTargetTbVgInfo) {
pRealTable->tsmaTargetTbVgInfo = taosArrayInit(pRealTable->pTsmas->size, POINTER_BYTES);
if (!pRealTable->tsmaTargetTbVgInfo) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
@ -3694,7 +3695,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo
}
pVgpsInfo->numOfVgroups = 1;
pVgpsInfo->vgroups[0] = vgInfo;
taosArrayPush(pRealTable->tsmaTargetCTbVgInfo, &pVgpsInfo);
taosArrayPush(pRealTable->tsmaTargetTbVgInfo, &pVgpsInfo);
} else {
break;
}
@ -3705,23 +3706,25 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo
.requestId = pCxt->pParseCxt->requestId,
.requestObjRefId = pCxt->pParseCxt->requestRid,
.mgmtEps = pCxt->pParseCxt->mgmtEpSet};
code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, &conn, &tsmaTargetCTbName, &pTableMeta);
code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, &conn, &tsmaTargetTbName, &pTableMeta);
}
if (code == TSDB_CODE_SUCCESS) {
STsmaTargetCTbInfo ctbInfo = {0};
sprintf(ctbInfo.ctableName, "%s", tsmaTargetCTbName.tname);
ctbInfo.uid = pTableMeta->uid;
taosMemoryFree(pTableMeta);
if (!pRealTable->tsmaTargetCTbInfo) {
pRealTable->tsmaTargetCTbInfo = taosArrayInit(pRealTable->pTsmas->size, sizeof(STsmaTargetCTbInfo));
if (!pRealTable->tsmaTargetCTbInfo) {
STsmaTargetTbInfo ctbInfo = {0};
if (!pRealTable->tsmaTargetTbInfo) {
pRealTable->tsmaTargetTbInfo = taosArrayInit(pRealTable->pTsmas->size, sizeof(STsmaTargetTbInfo));
if (!pRealTable->tsmaTargetTbInfo) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
}
taosArrayPush(pRealTable->tsmaTargetCTbInfo, &ctbInfo);
if (code == TSDB_CODE_SUCCESS) {
sprintf(ctbInfo.tableName, "%s", tsmaTargetTbName.tname);
ctbInfo.uid = pTableMeta->uid;
taosMemoryFree(pTableMeta);
} else if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) {
// ignore table not exists error
code = TSDB_CODE_SUCCESS;
}
taosArrayPush(pRealTable->tsmaTargetTbInfo, &ctbInfo);
}
}
}
@ -5681,8 +5684,8 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt*
vgsInfo = NULL;
if (pInfo->pRealTable->pTsmas) {
pInfo->pRealTable->tsmaTargetCTbVgInfo = taosArrayInit(pInfo->pRealTable->pTsmas->size, POINTER_BYTES);
if (!pInfo->pRealTable->tsmaTargetCTbVgInfo) return TSDB_CODE_OUT_OF_MEMORY;
pInfo->pRealTable->tsmaTargetTbVgInfo = taosArrayInit(pInfo->pRealTable->pTsmas->size, POINTER_BYTES);
if (!pInfo->pRealTable->tsmaTargetTbVgInfo) return TSDB_CODE_OUT_OF_MEMORY;
for (int32_t i = 0; i < pInfo->pRealTable->pTsmas->size; ++i) {
STableTSMAInfo* pTsma = taosArrayGetP(pInfo->pRealTable->pTsmas, i);
@ -5708,7 +5711,7 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt*
if (TSDB_CODE_SUCCESS == code) {
findVgroupsFromEqualTbname(pCxt, pTbNames, pInfo->pRealTable->table.dbName, numOfVgs, vgsInfo);
if (vgsInfo->numOfVgroups != 0) {
taosArrayPush(pInfo->pRealTable->tsmaTargetCTbVgInfo, &vgsInfo);
taosArrayPush(pInfo->pRealTable->tsmaTargetTbVgInfo, &vgsInfo);
} else {
taosMemoryFree(vgsInfo);
}

4
source/libs/planner/src/planLogicCreater.c
View File

@ -365,8 +365,8 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT
TSWAP(pScan->pVgroupList, pRealTable->pVgroupList);
TSWAP(pScan->pSmaIndexes, pRealTable->pSmaIndexes);
TSWAP(pScan->pTsmas, pRealTable->pTsmas);
TSWAP(pScan->pTsmaTargetCTbVgInfo, pRealTable->tsmaTargetCTbVgInfo);
TSWAP(pScan->pTsmaTargetCTbInfo, pRealTable->tsmaTargetCTbInfo);
TSWAP(pScan->pTsmaTargetTbVgInfo, pRealTable->tsmaTargetTbVgInfo);
TSWAP(pScan->pTsmaTargetTbInfo, pRealTable->tsmaTargetTbInfo);
pScan->tableId = pRealTable->pMeta->uid;
pScan->stableId = pRealTable->pMeta->suid;
pScan->tableType = pRealTable->pMeta->tableType;

22
source/libs/planner/src/planOptimizer.c
View File

@ -6287,7 +6287,7 @@ static int32_t tsmaOptRewriteScan(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pNew
pNewScan->pScanCols = tsmaOptCreateTsmaScanCols(pTsma, pTsmaOptCtx->pAggFuncs);
if (!pNewScan->pScanCols) code = TSDB_CODE_OUT_OF_MEMORY;
}
if (pNewScan->tableType == TSDB_CHILD_TABLE) {
if (pNewScan->tableType == TSDB_CHILD_TABLE) { // TODO remvoe it
}
if (code == TSDB_CODE_SUCCESS && pPkTsCol) {
tstrncpy(pPkTsCol->tableName, pTsma->targetTbName, TSDB_TABLE_NAME_LEN);
@ -6310,11 +6310,11 @@ static int32_t tsmaOptRewriteScan(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pNew
if (code == TSDB_CODE_SUCCESS) {
code = tsmaOptRewriteNodeList(pNewScan->pGroupTags, pTsmaOptCtx, pTsma, true, true);
}
if (pNewScan->pTsmaTargetCTbVgInfo && pNewScan->pTsmaTargetCTbVgInfo->size > 0) {
for (int32_t i = 0; i < taosArrayGetSize(pNewScan->pTsmas); ++i) {
STableTSMAInfo* pTsmaInfo = taosArrayGetP(pNewScan->pTsmas, i);
if (pTsmaOptCtx->pScan->pTsmaTargetTbVgInfo && pTsmaOptCtx->pScan->pTsmaTargetTbVgInfo->size > 0) {
for (int32_t i = 0; i < taosArrayGetSize(pTsmaOptCtx->pScan->pTsmas); ++i) {
STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmaOptCtx->pScan->pTsmas, i);
if (pTsmaInfo == pTsma->pTsma) {
SVgroupsInfo* pVgpsInfo = taosArrayGetP(pNewScan->pTsmaTargetCTbVgInfo, i);
const SVgroupsInfo* pVgpsInfo = taosArrayGetP(pTsmaOptCtx->pScan->pTsmaTargetTbVgInfo, i);
taosMemoryFreeClear(pNewScan->pVgroupList);
int32_t len = sizeof(int32_t) + sizeof(SVgroupInfo) * pVgpsInfo->numOfVgroups;
pNewScan->pVgroupList = taosMemoryCalloc(1, len);
@ -6533,13 +6533,14 @@ static int32_t tsmaOptGeneratePlan(STSMAOptCtx* pTsmaOptCtx) {
// TODO if no used tsmas skip generating plans
for (int32_t i = 0; i < pTsmaOptCtx->pUsedTsmas->size; ++i) {
STSMAOptUsefulTsma* pTsma = taosArrayGet(pTsmaOptCtx->pUsedTsmas, i);
if (pTsma->pTsma) {
if (pTsmaOptCtx->pScan->tableType == TSDB_CHILD_TABLE) {
if (!pTsma->pTsma) continue;
if (pTsmaOptCtx->pScan->tableType == TSDB_CHILD_TABLE || pTsmaOptCtx->pScan->tableType == TSDB_NORMAL_TABLE) {
for (int32_t j = 0; j < pTsmaOptCtx->pScan->pTsmas->size; ++j) {
if (taosArrayGetP(pTsmaOptCtx->pScan->pTsmas, j) == pTsma->pTsma) {
const STsmaTargetCTbInfo* pCtbInfo = taosArrayGet(pTsmaOptCtx->pScan->pTsmaTargetCTbInfo, j);
strcpy(pTsma->targetTbName, pCtbInfo->ctableName);
pTsma->targetTbUid = pCtbInfo->uid;
const STsmaTargetTbInfo* ptbInfo = taosArrayGet(pTsmaOptCtx->pScan->pTsmaTargetTbInfo, j);
ASSERT(ptbInfo->uid != 0);
strcpy(pTsma->targetTbName, ptbInfo->tableName);
pTsma->targetTbUid = ptbInfo->uid;
}
}
} else {
@ -6547,7 +6548,6 @@ static int32_t tsmaOptGeneratePlan(STSMAOptCtx* pTsmaOptCtx) {
pTsma->targetTbUid = pTsma->pTsma->destTbUid;
}
}
}
for (int32_t i = 1; i < pTsmaOptCtx->pUsedTsmas->size && code == TSDB_CODE_SUCCESS; ++i) {
pTsma = taosArrayGet(pTsmaOptCtx->pUsedTsmas, i);

16
tests/system-test/2-query/tsma.py
View File

@ -339,7 +339,8 @@ class TSMATestSQLGenerator:
else:
return auto_order_by[:-1]
def generate_one(self, select_list: str, tb: str, order_by_list: str, interval_list: List[str] = []) -> str:
def generate_one(self, select_list: str, possible_tbs: List, order_by_list: str, interval_list: List[str] = []) -> str:
tb = random.choice(possible_tbs)
where = self.generate_where()
interval = self.generate_interval(interval_list)
(partition_by, partition_by_list) = self.generate_partition_by()
@ -492,7 +493,7 @@ class TSMATestSQLGenerator:
class TDTestCase:
updatecfgDict = {'debugFlag': 143, 'asynclog': 0}
def __init__(self):
self.vgroups = 1
self.vgroups = 4
self.ctbNum = 10
self.rowsPerTbl = 10000
self.duraion = '1h'
@ -610,6 +611,8 @@ class TDTestCase:
if ctx.has_tsma():
if ctx.used_tsmas[0].name == tsma_name + UsedTsma.TSMA_RES_STB_POSTFIX:
break
elif ctx.used_tsmas[0].name.find('_') == 32 and ctx.used_tsmas[0].name[33:] == tb:
break
else:
time.sleep(1)
else:
@ -720,12 +723,17 @@ class TDTestCase:
interval_list = ['1s', '5s', '60s', '1m', '10m', '20m', '30m', '59s', '1h', '120s', '1200', '2h', '90m', '1d']
opts: TSMATesterSQLGeneratorOptions = TSMATesterSQLGeneratorOptions()
opts.partition_by = True
opts.interval = True
opts.where_ts_range = True
for _ in range(1, 100):
opts.partition_by = True
sql_generator = TSMATestSQLGenerator(opts)
sql = sql_generator.generate_one('avg(c1), avg(c2)', 'meters', '', interval_list)
sql = sql_generator.generate_one('avg(c1), avg(c2)', ['meters', 't1', 't9'], '', interval_list)
ctxs.append(TSMAQCBuilder().with_sql(sql).ignore_query_table().ignore_res_order(sql_generator.can_ignore_res_order()).get_qc())
opts.partition_by = False
sql_generator = TSMATestSQLGenerator(opts)
sql = sql_generator.generate_one('avg(c1), avg(c2)', ['norm_tb', 't5'], '', interval_list)
ctxs.append(TSMAQCBuilder().with_sql(sql).ignore_query_table().ignore_res_order(sql_generator.can_ignore_res_order()).get_qc())
return ctxs