xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

TD-14396 explain parser and planner implement

This commit is contained in:
Xiaoyu Wang 2022-03-30 02:50:50 -04:00
parent c439f2fb37
commit 03771c84f5
16 changed files with 1965 additions and 1772 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
include/common/ttokendef.h
View File

@ -138,55 +138,59 @@
#define TK_INTERVAL 120
#define TK_TOPIC 121
#define TK_AS 122
#define TK_NK_BOOL 123
#define TK_NULL 124
#define TK_NK_VARIABLE 125
#define TK_NK_UNDERLINE 126
#define TK_ROWTS 127
#define TK_TBNAME 128
#define TK_QSTARTTS 129
#define TK_QENDTS 130
#define TK_WSTARTTS 131
#define TK_WENDTS 132
#define TK_WDURATION 133
#define TK_BETWEEN 134
#define TK_IS 135
#define TK_NK_LT 136
#define TK_NK_GT 137
#define TK_NK_LE 138
#define TK_NK_GE 139
#define TK_NK_NE 140
#define TK_MATCH 141
#define TK_NMATCH 142
#define TK_IN 143
#define TK_JOIN 144
#define TK_INNER 145
#define TK_SELECT 146
#define TK_DISTINCT 147
#define TK_WHERE 148
#define TK_PARTITION 149
#define TK_BY 150
#define TK_SESSION 151
#define TK_STATE_WINDOW 152
#define TK_SLIDING 153
#define TK_FILL 154
#define TK_VALUE 155
#define TK_NONE 156
#define TK_PREV 157
#define TK_LINEAR 158
#define TK_NEXT 159
#define TK_GROUP 160
#define TK_HAVING 161
#define TK_ORDER 162
#define TK_SLIMIT 163
#define TK_SOFFSET 164
#define TK_LIMIT 165
#define TK_OFFSET 166
#define TK_ASC 167
#define TK_DESC 168
#define TK_NULLS 169
#define TK_FIRST 170
#define TK_LAST 171
#define TK_EXPLAIN 123
#define TK_ANALYZE 124
#define TK_VERBOSE 125
#define TK_NK_BOOL 126
#define TK_RATIO 127
#define TK_NULL 128
#define TK_NK_VARIABLE 129
#define TK_NK_UNDERLINE 130
#define TK_ROWTS 131
#define TK_TBNAME 132
#define TK_QSTARTTS 133
#define TK_QENDTS 134
#define TK_WSTARTTS 135
#define TK_WENDTS 136
#define TK_WDURATION 137
#define TK_BETWEEN 138
#define TK_IS 139
#define TK_NK_LT 140
#define TK_NK_GT 141
#define TK_NK_LE 142
#define TK_NK_GE 143
#define TK_NK_NE 144
#define TK_MATCH 145
#define TK_NMATCH 146
#define TK_IN 147
#define TK_JOIN 148
#define TK_INNER 149
#define TK_SELECT 150
#define TK_DISTINCT 151
#define TK_WHERE 152
#define TK_PARTITION 153
#define TK_BY 154
#define TK_SESSION 155
#define TK_STATE_WINDOW 156
#define TK_SLIDING 157
#define TK_FILL 158
#define TK_VALUE 159
#define TK_NONE 160
#define TK_PREV 161
#define TK_LINEAR 162
#define TK_NEXT 163
#define TK_GROUP 164
#define TK_HAVING 165
#define TK_ORDER 166
#define TK_SLIMIT 167
#define TK_SOFFSET 168
#define TK_LIMIT 169
#define TK_OFFSET 170
#define TK_ASC 171
#define TK_DESC 172
#define TK_NULLS 173
#define TK_FIRST 174
#define TK_LAST 175
#define TK_NK_SPACE 300
#define TK_NK_COMMENT 301

2
include/libs/nodes/nodes.h
View File

@ -70,6 +70,7 @@ typedef enum ENodeType {
QUERY_NODE_DATABASE_OPTIONS,
QUERY_NODE_TABLE_OPTIONS,
QUERY_NODE_INDEX_OPTIONS,
QUERY_NODE_EXPLAIN_OPTIONS,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR,
@ -99,6 +100,7 @@ typedef enum ENodeType {
QUERY_NODE_CREATE_TOPIC_STMT,
QUERY_NODE_DROP_TOPIC_STMT,
QUERY_NODE_ALTER_LOCAL_STMT,
QUERY_NODE_EXPLAIN_STMT,
QUERY_NODE_SHOW_DATABASES_STMT,
QUERY_NODE_SHOW_TABLES_STMT,
QUERY_NODE_SHOW_STABLES_STMT,

19
include/libs/nodes/plannodes.h
View File

@ -49,6 +49,7 @@ typedef struct SScanLogicNode {
STimeWindow scanRange;
SName tableName;
bool showRewrite;
double ratio;
} SScanLogicNode;
typedef struct SJoinLogicNode {
@ -197,6 +198,7 @@ typedef struct STableScanPhysiNode {
SScanPhysiNode scan;
uint8_t scanFlag; // denotes reversed scan of data or not
STimeWindow scanRange;
double ratio;
} STableScanPhysiNode;
typedef STableScanPhysiNode STableSeqScanPhysiNode;
@ -297,18 +299,23 @@ typedef struct SSubplan {
SDataSinkNode* pDataSink; // data of the subplan flow into the datasink
} SSubplan;
typedef enum EQueryMode {
QUERY_MODE_NORMAL = 1,
QUERY_MODE_EXPLAIN,
QUERY_MODE_EXPLAIN_AN
} EQueryMode;
typedef enum EExplainMode {
EXPLAIN_MODE_DISABLE = 1,
EXPLAIN_MODE_STATIC,
EXPLAIN_MODE_ANALYZE
} EExplainMode;
typedef struct SExplainInfo {
EExplainMode mode;
bool verbose;
} SExplainInfo;
typedef struct SQueryPlan {
ENodeType type;
uint64_t queryId;
int32_t numOfSubplans;
SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
SExplainInfo explainInfo;
} SQueryPlan;
#ifdef __cplusplus

14
include/libs/nodes/querynodes.h
View File

@ -131,6 +131,7 @@ typedef struct SRealTableNode {
struct STableMeta* pMeta;
SVgroupsInfo* pVgroupList;
char useDbName[TSDB_DB_NAME_LEN];
double ratio;
} SRealTableNode;
typedef struct STempTableNode {
@ -282,6 +283,19 @@ typedef struct SVnodeModifOpStmt {
const char* sql; // current sql statement position
} SVnodeModifOpStmt;
typedef struct SExplainOptions {
ENodeType type;
bool verbose;
double ratio;
} SExplainOptions;
typedef struct SExplainStmt {
ENodeType type;
bool analyze;
SExplainOptions* pOptions;
SNode* pQuery;
} SExplainStmt;
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);

8
include/util/tdef.h
View File

@ -381,6 +381,14 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_DB_DELAY 10
#define TSDB_DEFAULT_DB_DELAY 2
#define TSDB_DEFAULT_EXPLAIN_VERBOSE false
#define TSDB_MIN_EXPLAIN_RATIO 0
#define TSDB_MAX_EXPLAIN_RATIO 1
#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001
#define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024
#define TSDB_MAX_JOIN_TABLE_NUM 10
#define TSDB_MAX_UNION_CLAUSE 5

4
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -82,6 +82,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(STableOptions));
case QUERY_NODE_INDEX_OPTIONS:
return makeNode(type, sizeof(SIndexOptions));
case QUERY_NODE_EXPLAIN_OPTIONS:
return makeNode(type, sizeof(SExplainOptions));
case QUERY_NODE_SET_OPERATOR:
return makeNode(type, sizeof(SSetOperator));
case QUERY_NODE_SELECT_STMT:
@ -132,6 +134,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCreateTopicStmt));
case QUERY_NODE_DROP_TOPIC_STMT:
return makeNode(type, sizeof(SDropTopicStmt));
case QUERY_NODE_EXPLAIN_STMT:
return makeNode(type, sizeof(SExplainStmt));
case QUERY_NODE_SHOW_DATABASES_STMT:
case QUERY_NODE_SHOW_TABLES_STMT:
case QUERY_NODE_SHOW_STABLES_STMT:

4
source/libs/parser/inc/parAst.h
View File

@ -159,6 +159,10 @@ SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId);
SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName);
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName);
SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue);
SNode* createDefaultExplainOptions(SAstCreateContext* pCxt);
SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal);
SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal);
SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions, SNode* pQuery);
#ifdef __cplusplus
}

12
source/libs/parser/inc/sql.y
View File

@ -339,6 +339,18 @@ cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C).
cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS db_name(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, &C); }
cmd ::= DROP TOPIC exists_opt(A) topic_name(B). { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); }
/************************************************ select **************************************************************/
cmd ::= EXPLAIN analyze_opt(A) explain_options(B) query_expression(C). { pCxt->pRootNode = createExplainStmt(pCxt, A, B, C); }
%type analyze_opt { bool }
%destructor analyze_opt { }
analyze_opt(A) ::= . { A = false; }
analyze_opt(A) ::= ANALYZE. { A = true; }
explain_options(A) ::= . { A = createDefaultExplainOptions(pCxt); }
explain_options(A) ::= explain_options(B) VERBOSE NK_BOOL(C). { A = setExplainVerbose(pCxt, B, &C); }
explain_options(A) ::= explain_options(B) RATIO NK_FLOAT(C). { A = setExplainRatio(pCxt, B, &C); }
/************************************************ select **************************************************************/
cmd ::= query_expression(A). { pCxt->pRootNode = A; }

27
source/libs/parser/src/parAstCreater.c
View File

@ -1316,3 +1316,30 @@ SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, cons
}
return (SNode*)pStmt;
}
SNode* createDefaultExplainOptions(SAstCreateContext* pCxt) {
SExplainOptions* pOptions = nodesMakeNode(QUERY_NODE_EXPLAIN_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->verbose = TSDB_DEFAULT_EXPLAIN_VERBOSE;
pOptions->ratio = TSDB_DEFAULT_EXPLAIN_RATIO;
return (SNode*)pOptions;
}
SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal) {
((SExplainOptions*)pOptions)->verbose = (0 == strncasecmp(pVal->z, "true", pVal->n));
return pOptions;
}
SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal) {
((SExplainOptions*)pOptions)->ratio = strtod(pVal->z, NULL);
return pOptions;
}
SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions, SNode* pQuery) {
SExplainStmt* pStmt = nodesMakeNode(QUERY_NODE_EXPLAIN_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->analyze = analyze;
pStmt->pOptions = (SExplainOptions*)pOptions;
pStmt->pQuery = pQuery;
return (SNode*)pStmt;
}

5
source/libs/parser/src/parTokenizer.c
View File

@ -31,6 +31,7 @@ static SKeyword keywordTable[] = {
{"ACCOUNT", TK_ACCOUNT},
{"ALL", TK_ALL},
{"ALTER", TK_ALTER},
{"ANALYZE", TK_ANALYZE},
{"AND", TK_AND},
{"AS", TK_AS},
{"ASC", TK_ASC},
@ -56,6 +57,7 @@ static SKeyword keywordTable[] = {
{"DOUBLE", TK_DOUBLE},
{"DROP", TK_DROP},
{"EXISTS", TK_EXISTS},
{"EXPLAIN", TK_EXPLAIN},
{"FILE_FACTOR", TK_FILE_FACTOR},
{"FILL", TK_FILL},
{"FLOAT", TK_FLOAT},
@ -110,6 +112,7 @@ static SKeyword keywordTable[] = {
{"QNODES", TK_QNODES},
{"QSTARTTS", TK_QSTARTTS},
{"QUORUM", TK_QUORUM},
{"RATIO", TK_RATIO},
{"REPLICA", TK_REPLICA},
{"RETENTIONS", TK_RETENTIONS},
{"ROLLUP", TK_ROLLUP},
@ -144,6 +147,7 @@ static SKeyword keywordTable[] = {
{"USING", TK_USING},
{"VALUES", TK_VALUES},
{"VARCHAR", TK_VARCHAR},
{"VERBOSE", TK_VERBOSE},
{"VGROUPS", TK_VGROUPS},
{"WAL", TK_WAL},
{"WDURATION", TK_WDURATION},
@ -224,7 +228,6 @@ static SKeyword keywordTable[] = {
// {"DETACH", TK_DETACH},
// {"EACH", TK_EACH},
// {"END", TK_END},
// {"EXPLAIN", TK_EXPLAIN},
// {"FAIL", TK_FAIL},
// {"FOR", TK_FOR},
// {"IGNORE", TK_IGNORE},

32
source/libs/parser/src/parTranslater.c
View File

@ -32,6 +32,7 @@ typedef struct STranslateContext {
SCmdMsgInfo* pCmdMsg;
SHashObj* pDbs;
SHashObj* pTables;
SExplainOptions* pExplainOpt;
} STranslateContext;
typedef struct SFullDatabaseName {
@ -673,6 +674,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
switch (nodeType(pTable)) {
case QUERY_NODE_REAL_TABLE: {
SRealTableNode* pRealTable = (SRealTableNode*)pTable;
pRealTable->ratio = (NULL != pCxt->pExplainOpt ? pCxt->pExplainOpt->ratio : 1.0);
SName name;
code = getTableMetaImpl(pCxt,
toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), &(pRealTable->pMeta));
@ -680,10 +682,9 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pRealTable->table.tableName);
}
code = setTableVgroupList(pCxt, &name, pRealTable);
if (TSDB_CODE_SUCCESS != code) {
return code;
if (TSDB_CODE_SUCCESS == code) {
code = addNamespace(pCxt, pRealTable);
}
code = addNamespace(pCxt, pRealTable);
break;
}
case QUERY_NODE_TEMP_TABLE: {
@ -1804,6 +1805,13 @@ static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pSt
return TSDB_CODE_SUCCESS;
}
static int32_t translateExplain(STranslateContext* pCxt, SExplainStmt* pStmt) {
if (pStmt->analyze) {
pCxt->pExplainOpt = pStmt->pOptions;
}
return translateQuery(pCxt, pStmt->pQuery);
}
static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pNode)) {
@ -1885,6 +1893,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_ALTER_LOCAL_STMT:
code = translateAlterLocal(pCxt, (SAlterLocalStmt*)pNode);
break;
case QUERY_NODE_EXPLAIN_STMT:
code = translateExplain(pCxt, (SExplainStmt*)pNode);
break;
default:
break;
}
@ -1903,7 +1914,11 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) {
}
int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) {
if (NULL != pRoot && QUERY_NODE_SELECT_STMT == nodeType(pRoot)) {
if (NULL == pRoot) {
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) {
SSelectStmt* pSelect = (SSelectStmt*) pRoot;
*numOfCols = LIST_LENGTH(pSelect->pProjectionList);
*pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema));
@ -1921,6 +1936,14 @@ int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** p
strcpy((*pSchema)[index].name, pExpr->aliasName);
index +=1;
}
} else if (QUERY_NODE_EXPLAIN_STMT == nodeType(pRoot)) {
*numOfCols = 1;
*pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema));
if (NULL == (*pSchema)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pSchema)[0].type = TSDB_DATA_TYPE_BINARY;
(*pSchema)[0].bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE;
}
return TSDB_CODE_SUCCESS;
@ -2505,6 +2528,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
switch (nodeType(pQuery->pRoot)) {
case QUERY_NODE_SELECT_STMT:
case QUERY_NODE_EXPLAIN_STMT:
pQuery->haveResultSet = true;
pQuery->directRpc = false;
pQuery->msgType = TDMT_VND_QUERY;

3463
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

13
source/libs/parser/test/parserAstTest.cpp
View File

@ -647,3 +647,16 @@ TEST_F(ParserTest, dropTopic) {
bind("drop topic if exists tp1");
ASSERT_TRUE(run());
}
TEST_F(ParserTest, explain) {
setDatabase("root", "test");
bind("explain SELECT * FROM t1");
ASSERT_TRUE(run());
bind("explain analyze SELECT * FROM t1");
ASSERT_TRUE(run());
bind("explain analyze verbose true ratio 0.01 SELECT * FROM t1");
ASSERT_TRUE(run());
}

3
source/libs/planner/src/planLogicCreater.c
View File

@ -197,6 +197,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
strcpy(pScan->tableName.dbname, pRealTable->table.dbName);
strcpy(pScan->tableName.tname, pRealTable->table.tableName);
pScan->showRewrite = pCxt->pPlanCxt->showRewrite;
pScan->ratio = pRealTable->ratio;
// set columns to scan
SNodeList* pCols = NULL;
@ -704,6 +705,8 @@ static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogi
return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt, pLogicNode);
case QUERY_NODE_VNODE_MODIF_STMT:
return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt, pLogicNode);
case QUERY_NODE_EXPLAIN_STMT:
return createQueryLogicNode(pCxt, ((SExplainStmt*)pStmt)->pQuery, pLogicNode);
default:
break;
}

16
source/libs/planner/src/planPhysiCreater.c
View File

@ -398,6 +398,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
pTableScan->scanFlag = pScanLogicNode->scanFlag;
pTableScan->scanRange = pScanLogicNode->scanRange;
pTableScan->ratio = pScanLogicNode->ratio;
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode);
pSubplan->execNodeStat.tableNum = pScanLogicNode->pVgroupList->vgroups[0].numOfTable;
@ -1095,6 +1096,16 @@ static void destoryPhysiPlanContext(SPhysiPlanContext* pCxt) {
taosArrayDestroyEx(pCxt->pLocationHelper, destoryLocationHash);
}
static void setExplainInfo(SPlanContext* pCxt, SQueryPlan* pPlan) {
if (QUERY_NODE_EXPLAIN_STMT == nodeType(pCxt->pAstRoot)) {
SExplainStmt* pStmt = (SExplainStmt*)pCxt->pAstRoot;
pPlan->explainInfo.mode = pStmt->analyze ? EXPLAIN_MODE_ANALYZE : EXPLAIN_MODE_STATIC;
pPlan->explainInfo.verbose = pStmt->pOptions->verbose;
} else {
pPlan->explainInfo.mode = EXPLAIN_MODE_DISABLE;
}
}
int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) {
SPhysiPlanContext cxt = {
.pPlanCxt = pCxt,
@ -1106,7 +1117,12 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP
if (NULL == cxt.pLocationHelper) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = doCreatePhysiPlan(&cxt, pLogicPlan, pPlan);
if (TSDB_CODE_SUCCESS == code) {
setExplainInfo(pCxt, *pPlan);
}
destoryPhysiPlanContext(&cxt);
return code;
}

13
source/libs/planner/test/plannerTest.cpp
View File

@ -282,3 +282,16 @@ TEST_F(PlannerTest, createSmaIndex) {
bind("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)");
ASSERT_TRUE(run());
}
TEST_F(PlannerTest, explain) {
setDatabase("root", "test");
bind("explain SELECT * FROM t1");
ASSERT_TRUE(run());
bind("explain analyze SELECT * FROM t1");
ASSERT_TRUE(run());
bind("explain analyze verbose true ratio 0.01 SELECT * FROM t1");
ASSERT_TRUE(run());
}