xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #23021 from taosdata/szhou/star-tbname 

feature: bi tools support star expansion to include tbname
This commit is contained in:
wade zhang 2023-09-25 12:22:36 +08:00 committed by GitHub
parent 5941d75849 61f006809f
commit 928c28fb68
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 288 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/libs/nodes/nodes.h
View File

@ -124,6 +124,7 @@ int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
int32_t nodesListPushFront(SNodeList* pList, SNode* pNode);
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
void nodesListInsertListAfterPos(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
SNode* nodesListGetNode(SNodeList* pList, int32_t index);
SListCell* nodesListGetCell(SNodeList* pList, int32_t index);
void nodesDestroyList(SNodeList* pList);

3
include/libs/nodes/querynodes.h
View File

@ -536,6 +536,9 @@ int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc);
const char* operatorTypeStr(EOperatorType type);
const char* logicConditionTypeStr(ELogicConditionType type);
bool nodesIsStar(SNode* pNode);
bool nodesIsTableStar(SNode* pNode);
#ifdef __cplusplus
}
#endif

2
include/libs/parser/parser.h
View File

@ -64,7 +64,7 @@ typedef struct SParseContext {
SArray* pTableMetaPos; // sql table pos => catalog data pos
SArray* pTableVgroupPos; // sql table pos => catalog data pos
int64_t allocatorId;
int32_t biMode;
int8_t biMode;
} SParseContext;
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);

30
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -1595,6 +1595,26 @@ void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc) {
nodesFree(pSrc);
}
void nodesListInsertListAfterPos(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc) {
if (NULL == pTarget || NULL == pPos || NULL == pSrc || NULL == pSrc->pHead) {
return;
}
if (NULL == pPos->pNext) {
pTarget->pTail = pSrc->pHead;
} else {
pPos->pNext->pPrev = pSrc->pHead;
}
pSrc->pHead->pPrev = pPos;
pSrc->pTail->pNext = pPos->pNext;
pPos->pNext = pSrc->pHead;
pTarget->length += pSrc->length;
nodesFree(pSrc);
}
SNode* nodesListGetNode(SNodeList* pList, int32_t index) {
SNode* node;
FOREACH(node, pList) {
@ -2318,3 +2338,13 @@ SValueNode* nodesMakeValueNodeFromBool(bool b) {
}
return pValNode;
}
bool nodesIsStar(SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0]) &&
(0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
}
bool nodesIsTableStar(SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) &&
(0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
}

3
source/libs/parser/CMakeLists.txt
View File

@ -1,4 +1,7 @@
aux_source_directory(src PARSER_SRC)
IF (TD_BI_SUPPORT)
LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/bi/src/biRewriteQuery.c)
ENDIF ()
add_library(parser STATIC ${PARSER_SRC})
target_include_directories(
parser

1
source/libs/parser/inc/parInt.h
View File

@ -36,7 +36,6 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS
int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery);
int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow);
int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow);
#ifdef __cplusplus
}
#endif

55
source/libs/parser/inc/parTranslater.h Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_PARSER_TRANS_H_
#define _TD_PARSER_TRANS_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "parToken.h"
#include "parUtil.h"
#include "parser.h"
typedef struct STranslateContext {
SParseContext* pParseCxt;
int32_t errCode;
SMsgBuf msgBuf;
SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode*
int32_t currLevel;
int32_t levelNo;
ESqlClause currClause;
SNode* pCurrStmt;
SCmdMsgInfo* pCmdMsg;
SHashObj* pDbs;
SHashObj* pTables;
SHashObj* pTargetTables;
SExplainOptions* pExplainOpt;
SParseMetaCache* pMetaCache;
bool createStream;
bool stableQuery;
bool showRewrite;
SNode* pPrevRoot;
SNode* pPostRoot;
} STranslateContext;
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect);
int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput);
#ifdef __cplusplus
}
#endif
#endif /*_TD_PARSER_TRANS_H_*/

65
source/libs/parser/src/parTranslater.c
View File

@ -14,6 +14,7 @@
*/
#include "parInt.h"
#include "parTranslater.h"
#include "catalog.h"
#include "cmdnodes.h"
@ -35,28 +36,6 @@ typedef struct SRewriteTbNameContext {
char* pTbName;
} SRewriteTbNameContext;
typedef struct STranslateContext {
SParseContext* pParseCxt;
int32_t errCode;
SMsgBuf msgBuf;
SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode*
int32_t currLevel;
int32_t levelNo;
ESqlClause currClause;
SNode* pCurrStmt;
SCmdMsgInfo* pCmdMsg;
SHashObj* pDbs;
SHashObj* pTables;
SHashObj* pTargetTables;
SExplainOptions* pExplainOpt;
SParseMetaCache* pMetaCache;
bool createStream;
bool stableQuery;
bool showRewrite;
SNode* pPrevRoot;
SNode* pPostRoot;
} STranslateContext;
typedef struct SBuildTopicContext {
bool colExists;
bool colNotFound;
@ -1419,7 +1398,7 @@ static EDealRes haveVectorFunction(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
static int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput) {
int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput) {
SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel);
size_t nums = taosArrayGetSize(pTables);
for (size_t i = 0; i < nums; ++i) {
@ -1810,17 +1789,8 @@ static int32_t translateBlockDistFunc(STranslateContext* pCtx, SFunctionNode* pF
return TSDB_CODE_SUCCESS;
}
static bool isStar(SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0]) &&
(0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
}
static bool isTableStar(SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) &&
(0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
}
static bool isStarParam(SNode* pNode) { return isStar(pNode) || isTableStar(pNode); }
static bool isStarParam(SNode* pNode) { return nodesIsStar(pNode) || nodesIsTableStar(pNode); }
static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
if (!fmIsMultiResFunc(pFunc->funcId)) {
@ -2857,7 +2827,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
return code;
}
static int32_t createAllColumns(STranslateContext* pCxt, bool igTags, SNodeList** pCols) {
static int32_t createAllColumns(STranslateContext* pCxt, bool igTags, SNodeList** pCols) {
*pCols = nodesMakeList();
if (NULL == *pCols) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY);
@ -2938,9 +2908,9 @@ static int32_t createMultiResFuncsParas(STranslateContext* pCxt, SNodeList* pSrc
SNodeList* pExprs = NULL;
SNode* pPara = NULL;
FOREACH(pPara, pSrcParas) {
if (isStar(pPara)) {
if (nodesIsStar(pPara)) {
code = createAllColumns(pCxt, true, &pExprs);
} else if (isTableStar(pPara)) {
} else if (nodesIsTableStar(pPara)) {
code = createTableAllCols(pCxt, (SColumnNode*)pPara, true, &pExprs);
} else {
code = nodesListMakeStrictAppend(&pExprs, nodesCloneNode(pPara));
@ -3022,11 +2992,18 @@ static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) {
return TSDB_CODE_SUCCESS;
}
#ifndef TD_ENTERPRISE
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
return TSDB_CODE_SUCCESS;
}
#endif
static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
SNode* pNode = NULL;
WHERE_EACH(pNode, pSelect->pProjectionList) {
int32_t code = TSDB_CODE_SUCCESS;
if (isStar(pNode)) {
if (nodesIsStar(pNode)) {
SNodeList* pCols = NULL;
code = createAllColumns(pCxt, false, &pCols);
if (TSDB_CODE_SUCCESS == code) {
@ -3046,7 +3023,7 @@ static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
ERASE_NODE(pSelect->pProjectionList);
continue;
}
} else if (isTableStar(pNode)) {
} else if (nodesIsTableStar(pNode)) {
SNodeList* pCols = NULL;
code = createTableAllCols(pCxt, (SColumnNode*)pNode, false, &pCols);
if (TSDB_CODE_SUCCESS == code) {
@ -3935,6 +3912,9 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect
if (TSDB_CODE_SUCCESS == code) {
code = translateHaving(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code && pCxt->pParseCxt->biMode != 0) {
code = biRewriteSelectStar(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateSelectList(pCxt, pSelect);
}
@ -9537,11 +9517,6 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
return TSDB_CODE_SUCCESS;
}
static int32_t rewriteQueryForBI(STranslateContext* pParseCxt, SQuery* pQuery) {
return TSDB_CODE_SUCCESS;
}
int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache) {
STranslateContext cxt = {0};
@ -9549,13 +9524,9 @@ int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMe
if (TSDB_CODE_SUCCESS == code) {
code = rewriteQuery(&cxt, pQuery);
}
if (TSDB_CODE_SUCCESS == code && pParseCxt->biMode != 0) {
code = rewriteQueryForBI(&cxt, pQuery);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateQuery(&cxt, pQuery->pRoot);
}
if (TSDB_CODE_SUCCESS == code && (cxt.pPrevRoot || cxt.pPostRoot)) {
pQuery->pPrevRoot = cxt.pPrevRoot;
pQuery->pPostRoot = cxt.pPostRoot;

1
tests/parallel_test/cases.task
View File

@ -1020,6 +1020,7 @@
,,y,script,./test.sh -f tsim/query/partitionby.sim
,,y,script,./test.sh -f tsim/query/tableCount.sim
,,y,script,./test.sh -f tsim/query/show_db_table_kind.sim
,,y,script,./test.sh -f tsim/query/bi_star_table.sim
,,y,script,./test.sh -f tsim/query/tag_scan.sim
,,y,script,./test.sh -f tsim/query/nullColSma.sim
,,y,script,./test.sh -f tsim/query/bug3398.sim

5
tests/script/tsim/parser/limit.sim
View File

@ -40,16 +40,19 @@ while $i < $halfNum
$c = $x / 10
$c = $c * 10
$c = $x - $c
$c2 = $c
$binary = 'binary . $c
$binary = $binary . '
$nchar = 'nchar . $c
$nchar = $nchar . '
$ts = $ts + $i
print insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar )
sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar )
$ts = $ts + $halfNum
sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar )
print insert into $tb1 values ( $ts , $c2 , NULL , $c2 , NULL , $c2 , $c2 , true, $binary , $nchar )
sql insert into $tb1 values ( $ts , $c2 , NULL , $c2 , NULL , $c2 , $c2 , true, $binary , $nchar )
$x = $x + 1
endw

8
tests/script/tsim/parser/limit_stb.sim
View File

@ -32,11 +32,13 @@ sql select * from $stb limit 5
if $rows != 5 then
return -1
endi
sql select * from $stb limit 5 offset 1
print select * from $stb order by ts limit 5 offset 1
sql select * from $stb order by ts limit 5 offset 1
if $rows != 5 then
return -1
endi
if $data01 != 1 then
print $data01 $data41
if $data01 != 0 then
return -1
endi
#if $data41 != 5 then
@ -48,10 +50,12 @@ if $rows != 5 then
return -1
endi
print select * from $stb order by ts desc limit 5 offset 1
sql select * from $stb order by ts desc limit 5 offset 1
if $rows != 5 then
return -1
endi
print $data01 $data11 $data41
if $data01 != 9 then
return -1
endi

111
tests/script/tsim/query/bi_star_table.sim Normal file
View File

@ -0,0 +1,111 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
sql drop database if exists db1;
sql create database db1 vgroups 3;
sql create database db1;
sql use db1;
sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create table tba1 using sta tags(1, 1, 1);
sql create table tba2 using sta tags(2, 2, 2);
sql insert into tba1 values(now, 1, "1");
sql insert into tba2 values(now + 1s, 2, "2");
sql create table tbn1 (ts timestamp, f1 int);
sql create database db2 vgroups 3;
sql create database db2;
sql use db2;
sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create table tba1 using sta tags(1, 1, 1);
sql create table tba2 using sta tags(2, 2, 2);
set_bi_mode 1
sql select * from db1.sta order by ts;
if $cols != 7 then
return -1
endi
if $data06 != tba1 then
return -1
endi
sql select last(*) from db1.sta;
if $cols != 4 then
return -1
endi
if $data03 != tba2 then
return -1
endi
sql select last_row(*) from db1.sta;
if $cols != 4 then
return -1
endi
if $data03 != tba2 then
return -1
endi
sql select first(*) from db1.sta;
if $cols != 4 then
return -1
endi
if $data03 != tba1 then
return -1
endi
print "=====table star ====================="
sql select b.* from db1.sta b order by ts;
if $cols != 7 then
return -1
endi
if $data06 != tba1 then
return -1
endi
sql select last(b.*) from db1.sta b;
if $cols != 4 then
return -1
endi
if $data03 != tba2 then
return -1
endi
sql select last_row(b.*) from db1.sta b;
if $cols != 4 then
return -1
endi
if $data03 != tba2 then
return -1
endi
sql select first(b.*) from db1.sta b;
if $cols != 4 then
return -1
endi
if $data03 != tba1 then
return -1
endi
sql select * from (select f1 from db1.sta);
if $cols != 1 then
return -1
endi
set_bi_mode 0
sql select * from db1.sta order by ts;
if $cols != 6 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

3
utils/tsim/inc/simInt.h
View File

@ -123,6 +123,7 @@ enum {
SIM_CMD_RETURN,
SIM_CMD_LINE_INSERT,
SIM_CMD_LINE_INSERT_ERROR,
SIM_CMD_SET_BI_MODE,
SIM_CMD_END
};
@ -161,6 +162,7 @@ typedef struct _script_t {
bool killed;
void *taos;
char rows[12]; // number of rows data retrieved
char cols[12]; // number of columns data retrieved
char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results
char system_exit_code[12];
char system_ret_content[MAX_SYSTEM_RESULT_LEN];
@ -210,6 +212,7 @@ bool simExecuteSqlSlowCmd(SScript *script, char *option);
bool simExecuteRestfulCmd(SScript *script, char *rest);
bool simExecuteLineInsertCmd(SScript *script, char *option);
bool simExecuteLineInsertErrorCmd(SScript *script, char *option);
bool simExecuteSetBIModeCmd(SScript *script, char *option);
void simVisuallizeOption(SScript *script, char *src, char *dst);
#endif /*_TD_SIM_INT_H_*/

25
utils/tsim/src/simExe.c
View File

@ -93,6 +93,8 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
if (strncmp(varName, "rows", varLen) == 0) return script->rows;
if (strncmp(varName, "cols", varLen) == 0) return script->cols;
if (strncmp(varName, "system_exit", varLen) == 0) return script->system_exit_code;
if (strncmp(varName, "system_content", varLen) == 0) return script->system_ret_content;
@ -502,6 +504,26 @@ bool simExecuteSystemContentCmd(SScript *script, char *option) {
return true;
}
bool simExecuteSetBIModeCmd(SScript *script, char *option) {
char buf[1024];
simVisuallizeOption(script, option, buf);
option = buf;
int32_t mode = atoi(option);
simInfo("script:%s, set bi mode %d", script->fileName, mode);
if (mode != 0) {
taos_set_conn_mode(script->taos, TAOS_CONN_MODE_BI, 1);
} else {
taos_set_conn_mode(script->taos, TAOS_CONN_MODE_BI, 0);
}
script->linePos++;
return true;
}
bool simExecutePrintCmd(SScript *script, char *rest) {
char buf[65536];
@ -808,6 +830,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
taos_free_result(pSql);
sprintf(script->rows, "%d", numOfRows);
sprintf(script->cols, "%d", num_fields);
script->linePos++;
return true;
@ -822,6 +845,7 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) {
simDebug("script:%s, exec:%s", script->fileName, rest);
strcpy(script->rows, "-1");
strcpy(script->cols, "-1");
for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) {
for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) {
strcpy(script->data[row][col], "null");
@ -918,6 +942,7 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) {
simDebug("script:%s, exec:%s", script->fileName, rest);
strcpy(script->rows, "-1");
strcpy(script->cols, "-1");
for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) {
for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) {
strcpy(script->data[row][col], "null");

27
utils/tsim/src/simParse.c
View File

@ -745,6 +745,25 @@ bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
return true;
}
bool simParseSetBIModeCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
char *token;
int32_t tokenLen;
cmdLine[numOfLines].cmdno = SIM_CMD_SET_BI_MODE;
cmdLine[numOfLines].lineNum = lineNum;
paGetToken(rest, &token, &tokenLen);
if (tokenLen > 0) {
cmdLine[numOfLines].optionOffset = optionOffset;
memcpy(optionBuffer + optionOffset, token, tokenLen);
optionOffset += tokenLen + 1;
*(optionBuffer + optionOffset - 1) = 0;
}
numOfLines++;
return true;
}
bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
char *token;
int32_t tokenLen;
@ -1074,6 +1093,14 @@ void simInitsimCmdList() {
simCmdList[cmdno].executeCmd = simExecuteReturnCmd;
simAddCmdIntoHash(&(simCmdList[cmdno]));
cmdno = SIM_CMD_SET_BI_MODE;
simCmdList[cmdno].cmdno = cmdno;
strcpy(simCmdList[cmdno].name, "set_bi_mode");
simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name);
simCmdList[cmdno].parseCmd = simParseSetBIModeCmd;
simCmdList[cmdno].executeCmd = simExecuteSetBIModeCmd;
simAddCmdIntoHash(&(simCmdList[cmdno]));
#if 0
cmdno = SIM_CMD_LINE_INSERT;
simCmdList[cmdno].cmdno = cmdno;