xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh: ins_tables optimize

This commit is contained in:
Xiaoyu Wang 2022-11-21 18:44:04 +08:00
parent 584ce8a9b8
commit 469fcd314e
11 changed files with 353 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/libs/nodes/nodes.h
View File

@ -60,6 +60,12 @@ extern "C" {
for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \
(NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext)
#define NODES_DESTORY_NODE(node) \
do { \
nodesDestroyNode((node)); \
(node) = NULL; \
} while (0)
#define NODES_DESTORY_LIST(list) \
do { \
nodesDestroyList((list)); \

2
source/libs/parser/inc/parUtil.h
View File

@ -86,7 +86,7 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta);
STableMeta* tableMetaDup(const STableMeta* pTableMeta);
int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName);
int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName);
int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq);
int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache);

2
source/libs/parser/src/parAstParser.c
View File

@ -140,7 +140,7 @@ static int32_t collectMetaKeyFromInsTagsImpl(SCollectMetaKeyCxt* pCxt, SName* pN
static int32_t collectMetaKeyFromInsTags(SCollectMetaKeyCxt* pCxt) {
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pStmt;
SName name = {0};
int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name);
int32_t code = getVnodeSysTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name);
if (TSDB_CODE_SUCCESS == code) {
code = collectMetaKeyFromInsTagsImpl(pCxt, &name);
}

201
source/libs/parser/src/parTranslater.c
View File

@ -2203,22 +2203,28 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) {
}
static bool sysTableFromVnode(const char* pTable) {
return (0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) ||
(0 == strcmp(pTable, TSDB_INS_TABLE_TABLE_DISTRIBUTED) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)));
return ((0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)));
}
static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); }
static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTargetName, SName* pName,
SArray** pVgroupList) {
static int32_t getVnodeSysTableVgroupListImpl(STranslateContext* pCxt, SName* pTargetName, SName* pName,
SArray** pVgroupList) {
if (0 == pTargetName->type) {
return getDBVgInfoImpl(pCxt, pName, pVgroupList);
}
if (0 == strcmp(pTargetName->dbname, TSDB_INFORMATION_SCHEMA_DB) ||
0 == strcmp(pTargetName->dbname, TSDB_PERFORMANCE_SCHEMA_DB)) {
pTargetName->type = 0;
return TSDB_CODE_SUCCESS;
}
if (TSDB_DB_NAME_T == pTargetName->type) {
int32_t code = getDBVgInfoImpl(pCxt, pTargetName, pVgroupList);
if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code ||
TSDB_CODE_MND_DB_IN_DROPPING == code) {
// system table query should not report errors
code = TSDB_CODE_SUCCESS;
}
return code;
@ -2235,50 +2241,44 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge
}
} else if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code ||
TSDB_CODE_MND_DB_IN_DROPPING == code) {
// system table query should not report errors
code = TSDB_CODE_SUCCESS;
}
return code;
}
static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) {
static int32_t getVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgs, bool* pHasUserDbCond) {
if (!isSelectStmt(pCxt->pCurrStmt)) {
return TSDB_CODE_SUCCESS;
}
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
SName targetName = {0};
int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName);
int32_t code = getVnodeSysTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName);
if (TSDB_CODE_SUCCESS == code) {
code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList);
code = getVnodeSysTableVgroupListImpl(pCxt, &targetName, pName, pVgs);
}
*pHasUserDbCond = (0 != targetName.type);
return code;
}
static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
int32_t code = TSDB_CODE_SUCCESS;
SArray* vgroupList = NULL;
if (0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS)) {
code = getTagsTableVgroupList(pCxt, pName, &vgroupList);
} else if ('\0' != pRealTable->qualDbName[0]) {
if (0 != strcmp(pRealTable->qualDbName, TSDB_INFORMATION_SCHEMA_DB)) {
code = getDBVgInfo(pCxt, pRealTable->qualDbName, &vgroupList);
}
} else {
code = getDBVgInfoImpl(pCxt, pName, &vgroupList);
}
bool hasUserDbCond = false;
SArray* pVgs = NULL;
int32_t code = getVnodeSysTableVgroupList(pCxt, pName, &pVgs, &hasUserDbCond);
if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) &&
isSelectStmt(pCxt->pCurrStmt) && 0 == taosArrayGetSize(vgroupList)) {
isSelectStmt(pCxt->pCurrStmt) && 0 == taosArrayGetSize(pVgs)) {
((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true;
}
if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES)) {
code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList);
if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) {
code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &pVgs);
}
if (TSDB_CODE_SUCCESS == code) {
code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList);
code = toVgroupsInfo(pVgs, &pRealTable->pVgroupList);
}
taosArrayDestroy(vgroupList);
taosArrayDestroy(pVgs);
return code;
}
@ -2303,30 +2303,39 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea
}
}
static int32_t setSuperTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
SArray* vgroupList = NULL;
int32_t code = getDBVgInfoImpl(pCxt, pName, &vgroupList);
if (TSDB_CODE_SUCCESS == code) {
code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList);
}
taosArrayDestroy(vgroupList);
return code;
}
static int32_t setNormalTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo));
if (NULL == pRealTable->pVgroupList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pRealTable->pVgroupList->numOfVgroups = 1;
return getTableHashVgroupImpl(pCxt, pName, pRealTable->pVgroupList->vgroups);
}
static int32_t setTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
if (pCxt->pParseCxt->topicQuery) {
return TSDB_CODE_SUCCESS;
}
int32_t code = TSDB_CODE_SUCCESS;
if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) {
SArray* vgroupList = NULL;
code = getDBVgInfoImpl(pCxt, pName, &vgroupList);
if (TSDB_CODE_SUCCESS == code) {
code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList);
}
taosArrayDestroy(vgroupList);
} else if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) {
code = setSysTableVgroupList(pCxt, pName, pRealTable);
} else {
pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo));
if (NULL == pRealTable->pVgroupList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pRealTable->pVgroupList->numOfVgroups = 1;
code = getTableHashVgroupImpl(pCxt, pName, pRealTable->pVgroupList->vgroups);
return setSuperTableVgroupList(pCxt, pName, pRealTable);
}
return code;
if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) {
return setSysTableVgroupList(pCxt, pName, pRealTable);
}
return setNormalTableVgroupList(pCxt, pName, pRealTable);
}
static uint8_t getStmtPrecision(SNode* pStmt) {
@ -2360,7 +2369,6 @@ static bool isSingleTable(SRealTableNode* pRealTable) {
int8_t tableType = pRealTable->pMeta->tableType;
if (TSDB_SYSTEM_TABLE == tableType) {
return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) &&
0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLE_DISTRIBUTED) &&
0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS);
}
return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType);
@ -7467,112 +7475,6 @@ static int32_t rewriteFlushDatabase(STranslateContext* pCxt, SQuery* pQuery) {
return code;
}
static bool isTableCountProject(SNodeList* pProjectionList) {
if (1 != LIST_LENGTH(pProjectionList)) {
return false;
}
SNode* pProj = nodesListGetNode(pProjectionList, 0);
return QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp(((SFunctionNode*)pProj)->functionName, "count");
}
static bool isTableCountFrom(SNode* pTable) {
if (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable)) {
return false;
}
SRealTableNode* pRtable = (SRealTableNode*)pTable;
return 0 == strcmp(pRtable->table.dbName, TSDB_INFORMATION_SCHEMA_DB) &&
0 == strcmp(pRtable->table.tableName, TSDB_INS_TABLE_TABLES);
}
static bool isTableCountCond(SNode* pCond, const char* pCol) {
if (QUERY_NODE_OPERATOR != nodeType(pCond) || OP_TYPE_EQUAL != ((SOperatorNode*)pCond)->opType) {
return false;
}
SNode* pLeft = ((SOperatorNode*)pCond)->pLeft;
SNode* pRight = ((SOperatorNode*)pCond)->pRight;
if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_VALUE == nodeType(pRight)) {
return 0 == strcmp(((SColumnNode*)pLeft)->colName, pCol);
}
if (QUERY_NODE_COLUMN == nodeType(pRight) && QUERY_NODE_VALUE == nodeType(pLeft)) {
return 0 == strcmp(((SColumnNode*)pRight)->colName, pCol);
}
return false;
}
static bool isTableCountWhere(SNode* pWhere) {
if (NULL == pWhere || QUERY_NODE_LOGIC_CONDITION != nodeType(pWhere)) {
return false;
}
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pWhere;
if (LOGIC_COND_TYPE_AND != pLogicCond->condType || 2 != LIST_LENGTH(pLogicCond->pParameterList)) {
return false;
}
SNode* pCond1 = nodesListGetNode(pLogicCond->pParameterList, 0);
SNode* pCond2 = nodesListGetNode(pLogicCond->pParameterList, 1);
return (isTableCountCond(pCond1, "db_name") && isTableCountCond(pCond2, "stable_name")) ||
(isTableCountCond(pCond1, "stable_name") && isTableCountCond(pCond2, "db_name"));
}
static bool isTableCountQuery(const SSelectStmt* pSelect) {
if (!isTableCountProject(pSelect->pProjectionList) || !isTableCountFrom(pSelect->pFromTable) ||
!isTableCountWhere(pSelect->pWhere) || NULL != pSelect->pPartitionByList || NULL != pSelect->pWindow ||
NULL != pSelect->pGroupByList || NULL != pSelect->pHaving || NULL != pSelect->pRange || NULL != pSelect->pEvery ||
NULL != pSelect->pFill) {
return false;
}
return true;
}
static SNode* createTableCountPseudoColumn() {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) {
return NULL;
}
snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", "_table_count");
return (SNode*)pFunc;
}
static int32_t rewriteCountFuncForTableCount(SSelectStmt* pSelect) {
SFunctionNode* pFunc = (SFunctionNode*)nodesListGetNode(pSelect->pProjectionList, 0);
NODES_DESTORY_LIST(pFunc->pParameterList);
snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", "sum");
return nodesListMakeStrictAppend(&pFunc->pParameterList, createTableCountPseudoColumn());
}
static const char* getNameFromCond(SLogicConditionNode* pLogicCond, const char* pCol) {
SOperatorNode* pCond1 = (SOperatorNode*)nodesListGetNode(pLogicCond->pParameterList, 0);
SOperatorNode* pCond2 = (SOperatorNode*)nodesListGetNode(pLogicCond->pParameterList, 1);
if (QUERY_NODE_COLUMN == nodeType(pCond1->pLeft) && 0 == strcmp(((SColumnNode*)pCond1->pLeft)->colName, pCol)) {
return ((SValueNode*)pCond1->pRight)->literal;
}
if (QUERY_NODE_COLUMN == nodeType(pCond1->pRight) && 0 == strcmp(((SColumnNode*)pCond1->pRight)->colName, pCol)) {
return ((SValueNode*)pCond1->pLeft)->literal;
}
if (QUERY_NODE_COLUMN == nodeType(pCond2->pLeft) && 0 == strcmp(((SColumnNode*)pCond2->pLeft)->colName, pCol)) {
return ((SValueNode*)pCond2->pRight)->literal;
}
return ((SValueNode*)pCond2->pLeft)->literal;
}
static int32_t rewriteRealTableForTableCount(SSelectStmt* pSelect) {
STableNode* pTable = (STableNode*)pSelect->pFromTable;
snprintf(pTable->dbName, sizeof(pTable->dbName), "%s",
getNameFromCond((SLogicConditionNode*)pSelect->pWhere, "db_name"));
snprintf(pTable->tableName, sizeof(pTable->tableName), "%s",
getNameFromCond((SLogicConditionNode*)pSelect->pWhere, "stable_name"));
nodesDestroyNode(pSelect->pWhere);
pSelect->pWhere = NULL;
return TSDB_CODE_SUCCESS;
}
static int32_t rewriteTableCountQuery(SSelectStmt* pSelect) {
int32_t code = rewriteCountFuncForTableCount(pSelect);
if (TSDB_CODE_SUCCESS == code) {
code = rewriteRealTableForTableCount(pSelect);
}
return code;
}
static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pQuery->pRoot)) {
@ -7633,11 +7535,6 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_FLUSH_DATABASE_STMT:
code = rewriteFlushDatabase(pCxt, pQuery);
break;
case QUERY_NODE_SELECT_STMT:
if (isTableCountQuery((SSelectStmt*)pQuery->pRoot)) {
code = rewriteTableCountQuery((SSelectStmt*)pQuery->pRoot);
}
break;
default:
break;
}

2
source/libs/parser/src/parUtil.c
View File

@ -474,7 +474,7 @@ static int32_t getInsTagsTableTargetNameFromCond(int32_t acctId, SLogicCondition
return TSDB_CODE_SUCCESS;
}
int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) {
int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) {
if (NULL == pWhere) {
return TSDB_CODE_SUCCESS;
}

12
source/libs/parser/test/mockCatalog.cpp
View File

@ -137,7 +137,7 @@ void generatePerformanceSchema(MockCatalogService* mcs) {
void generateTestTables(MockCatalogService* mcs, const std::string& db) {
mcs->createTableBuilder(db, "t1", TSDB_NORMAL_TABLE, 6)
.setPrecision(TSDB_TIME_PRECISION_MILLI)
.setVgid(1)
.setVgid(2)
.addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP)
.addColumn("c1", TSDB_DATA_TYPE_INT)
.addColumn("c2", TSDB_DATA_TYPE_BINARY, 20)
@ -179,9 +179,9 @@ void generateTestStables(MockCatalogService* mcs, const std::string& db) {
.addTag("tag2", TSDB_DATA_TYPE_BINARY, 20)
.addTag("tag3", TSDB_DATA_TYPE_TIMESTAMP);
builder.done();
mcs->createSubTable(db, "st1", "st1s1", 1);
mcs->createSubTable(db, "st1", "st1s2", 2);
mcs->createSubTable(db, "st1", "st1s3", 1);
mcs->createSubTable(db, "st1", "st1s1", 2);
mcs->createSubTable(db, "st1", "st1s2", 3);
mcs->createSubTable(db, "st1", "st1s3", 2);
}
{
ITableBuilder& builder = mcs->createTableBuilder(db, "st2", TSDB_SUPER_TABLE, 3, 1)
@ -191,8 +191,8 @@ void generateTestStables(MockCatalogService* mcs, const std::string& db) {
.addColumn("c2", TSDB_DATA_TYPE_BINARY, 20)
.addTag("jtag", TSDB_DATA_TYPE_JSON);
builder.done();
mcs->createSubTable(db, "st2", "st2s1", 1);
mcs->createSubTable(db, "st2", "st2s2", 2);
mcs->createSubTable(db, "st2", "st2s1", 2);
mcs->createSubTable(db, "st2", "st2s2", 3);
}
}

136
source/libs/parser/test/mockCatalogService.cpp
View File

@ -20,15 +20,18 @@
#include <map>
#include <set>
#include "systable.h"
#include "tdatablock.h"
#include "tname.h"
#include "ttypes.h"
using std::string;
std::unique_ptr<MockCatalogService> g_mockCatalogService;
class TableBuilder : public ITableBuilder {
public:
virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) {
virtual TableBuilder& addColumn(const string& name, int8_t type, int32_t bytes) {
assert(colId_ <= schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns);
SSchema* col = schema()->schema + (colId_ - 1);
col->type = type;
@ -142,27 +145,16 @@ class MockCatalogServiceImpl {
}
int32_t catalogGetDBVgList(const char* pDbFName, SArray** pVgList) const {
std::string dbFName(pDbFName);
DbMetaCache::const_iterator it = meta_.find(dbFName.substr(std::string(pDbFName).find_last_of('.') + 1));
if (meta_.end() == it) {
return TSDB_CODE_FAILED;
string dbName(string(pDbFName).substr(string(pDbFName).find_last_of('.') + 1));
if (0 == dbName.compare(TSDB_INFORMATION_SCHEMA_DB) || 0 == dbName.compare(TSDB_PERFORMANCE_SCHEMA_DB)) {
return catalogGetAllDBVgList(pVgList);
}
std::set<int32_t> vgSet;
*pVgList = taosArrayInit(it->second.size(), sizeof(SVgroupInfo));
for (const auto& vgs : it->second) {
for (const auto& vg : vgs.second->vgs) {
if (0 == vgSet.count(vg.vgId)) {
taosArrayPush(*pVgList, &vg);
vgSet.insert(vg.vgId);
}
}
}
return TSDB_CODE_SUCCESS;
return catalogGetDBVgListImpl(dbName, pVgList);
}
int32_t catalogGetDBCfg(const char* pDbFName, SDbCfgInfo* pDbCfg) const {
std::string dbFName(pDbFName);
DbCfgCache::const_iterator it = dbCfg_.find(dbFName.substr(std::string(pDbFName).find_last_of('.') + 1));
string dbFName(pDbFName);
DbCfgCache::const_iterator it = dbCfg_.find(dbFName.substr(string(pDbFName).find_last_of('.') + 1));
if (dbCfg_.end() == it) {
return TSDB_CODE_FAILED;
}
@ -171,7 +163,7 @@ class MockCatalogServiceImpl {
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const {
int32_t catalogGetUdfInfo(const string& funcName, SFuncInfo* pInfo) const {
auto it = udf_.find(funcName);
if (udf_.end() == it) {
return TSDB_CODE_FAILED;
@ -236,15 +228,15 @@ class MockCatalogServiceImpl {
return code;
}
TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType,
int32_t numOfColumns, int32_t numOfTags) {
TableBuilder& createTableBuilder(const string& db, const string& tbname, int8_t tableType, int32_t numOfColumns,
int32_t numOfTags) {
builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags);
meta_[db][tbname] = builder_->table();
meta_[db][tbname]->schema->uid = getNextId();
return *(builder_.get());
}
void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) {
void createSubTable(const string& db, const string& stbname, const string& tbname, int16_t vgid) {
std::unique_ptr<STableMeta> table;
if (TSDB_CODE_SUCCESS != copyTableSchemaMeta(db, stbname, &table)) {
throw std::runtime_error("copyTableSchemaMeta failed");
@ -274,13 +266,13 @@ class MockCatalogServiceImpl {
// string field length
#define SFL 20
// string field header
#define SH(h) CA(SFL, std::string(h))
#define SH(h) CA(SFL, string(h))
// string field
#define SF(n) CA(SFL, n)
// integer field length
#define IFL 10
// integer field header
#define IH(i) CA(IFL, std::string(i))
#define IH(i) CA(IFL, string(i))
// integer field
#define IF(i) CA(IFL, std::to_string(i))
// split line
@ -308,7 +300,7 @@ class MockCatalogServiceImpl {
int16_t numOfFields = numOfColumns + schema->tableInfo.numOfTags;
for (int16_t i = 0; i < numOfFields; ++i) {
const SSchema* col = schema->schema + i;
std::cout << SF(std::string(col->name)) << SH(ftToString(i, numOfColumns)) << SH(dtToString(col->type))
std::cout << SF(string(col->name)) << SH(ftToString(i, numOfColumns)) << SH(dtToString(col->type))
<< IF(col->bytes) << std::endl;
}
std::cout << std::endl;
@ -316,7 +308,7 @@ class MockCatalogServiceImpl {
}
}
void createFunction(const std::string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize) {
void createFunction(const string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize) {
std::shared_ptr<SFuncInfo> info(new SFuncInfo);
strcpy(info->name, func.c_str());
info->funcType = funcType;
@ -342,19 +334,19 @@ class MockCatalogServiceImpl {
info.expr = strdup(pReq->expr);
auto it = index_.find(pReq->stb);
if (index_.end() == it) {
index_.insert(std::make_pair(std::string(pReq->stb), std::vector<STableIndexInfo>{info}));
index_.insert(std::make_pair(string(pReq->stb), std::vector<STableIndexInfo>{info}));
} else {
it->second.push_back(info);
}
}
void createDnode(int32_t dnodeId, const std::string& host, int16_t port) {
void createDnode(int32_t dnodeId, const string& host, int16_t port) {
SEpSet epSet = {0};
addEpIntoEpSet(&epSet, host.c_str(), port);
dnode_.insert(std::make_pair(dnodeId, epSet));
}
void createDatabase(const std::string& db, bool rollup, int8_t cacheLast) {
void createDatabase(const string& db, bool rollup, int8_t cacheLast) {
SDbCfgInfo cfg = {0};
if (rollup) {
cfg.pRetensions = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SRetention));
@ -364,12 +356,12 @@ class MockCatalogServiceImpl {
}
private:
typedef std::map<std::string, std::shared_ptr<MockTableMeta>> TableMetaCache;
typedef std::map<std::string, TableMetaCache> DbMetaCache;
typedef std::map<std::string, std::shared_ptr<SFuncInfo>> UdfMetaCache;
typedef std::map<std::string, std::vector<STableIndexInfo>> IndexMetaCache;
typedef std::map<int32_t, SEpSet> DnodeCache;
typedef std::map<std::string, SDbCfgInfo> DbCfgCache;
typedef std::map<string, std::shared_ptr<MockTableMeta>> TableMetaCache;
typedef std::map<string, TableMetaCache> DbMetaCache;
typedef std::map<string, std::shared_ptr<SFuncInfo>> UdfMetaCache;
typedef std::map<string, std::vector<STableIndexInfo>> IndexMetaCache;
typedef std::map<int32_t, SEpSet> DnodeCache;
typedef std::map<string, SDbCfgInfo> DbCfgCache;
uint64_t getNextId() { return id_++; }
@ -386,15 +378,15 @@ class MockCatalogServiceImpl {
return pDst;
}
std::string toDbname(const std::string& dbFullName) const {
std::string::size_type n = dbFullName.find(".");
if (n == std::string::npos) {
string toDbname(const string& dbFullName) const {
string::size_type n = dbFullName.find(".");
if (n == string::npos) {
return dbFullName;
}
return dbFullName.substr(n + 1);
}
std::string ttToString(int8_t tableType) const {
string ttToString(int8_t tableType) const {
switch (tableType) {
case TSDB_SUPER_TABLE:
return "super table";
@ -407,7 +399,7 @@ class MockCatalogServiceImpl {
}
}
std::string pToString(uint8_t precision) const {
string pToString(uint8_t precision) const {
switch (precision) {
case TSDB_TIME_PRECISION_MILLI:
return "millisecond";
@ -420,19 +412,18 @@ class MockCatalogServiceImpl {
}
}
std::string dtToString(int8_t type) const { return tDataTypes[type].name; }
string dtToString(int8_t type) const { return tDataTypes[type].name; }
std::string ftToString(int16_t colid, int16_t numOfColumns) const {
string ftToString(int16_t colid, int16_t numOfColumns) const {
return (0 == colid ? "column" : (colid < numOfColumns ? "column" : "tag"));
}
STableMeta* getTableSchemaMeta(const std::string& db, const std::string& tbname) const {
STableMeta* getTableSchemaMeta(const string& db, const string& tbname) const {
std::shared_ptr<MockTableMeta> table = getTableMeta(db, tbname);
return table ? table->schema : nullptr;
}
int32_t copyTableSchemaMeta(const std::string& db, const std::string& tbname,
std::unique_ptr<STableMeta>* dst) const {
int32_t copyTableSchemaMeta(const string& db, const string& tbname, std::unique_ptr<STableMeta>* dst) const {
STableMeta* src = getTableSchemaMeta(db, tbname);
if (nullptr == src) {
return TSDB_CODE_TSC_INVALID_TABLE_NAME;
@ -446,7 +437,7 @@ class MockCatalogServiceImpl {
return TSDB_CODE_SUCCESS;
}
int32_t copyTableVgroup(const std::string& db, const std::string& tbname, SVgroupInfo* vg) const {
int32_t copyTableVgroup(const string& db, const string& tbname, SVgroupInfo* vg) const {
std::shared_ptr<MockTableMeta> table = getTableMeta(db, tbname);
if (table->vgs.empty()) {
return TSDB_CODE_SUCCESS;
@ -455,7 +446,7 @@ class MockCatalogServiceImpl {
return TSDB_CODE_SUCCESS;
}
int32_t copyTableVgroup(const std::string& db, const std::string& tbname, SArray** vgList) const {
int32_t copyTableVgroup(const string& db, const string& tbname, SArray** vgList) const {
std::shared_ptr<MockTableMeta> table = getTableMeta(db, tbname);
if (table->vgs.empty()) {
return TSDB_CODE_SUCCESS;
@ -467,7 +458,7 @@ class MockCatalogServiceImpl {
return TSDB_CODE_SUCCESS;
}
std::shared_ptr<MockTableMeta> getTableMeta(const std::string& db, const std::string& tbname) const {
std::shared_ptr<MockTableMeta> getTableMeta(const string& db, const string& tbname) const {
DbMetaCache::const_iterator it = meta_.find(db);
if (meta_.end() == it) {
return std::shared_ptr<MockTableMeta>();
@ -527,6 +518,40 @@ class MockCatalogServiceImpl {
return code;
}
int32_t catalogGetDBVgListImpl(const string& dbName, SArray** pVgList) const {
DbMetaCache::const_iterator it = meta_.find(dbName);
if (meta_.end() == it) {
return TSDB_CODE_FAILED;
}
std::set<int32_t> vgSet;
*pVgList = taosArrayInit(it->second.size(), sizeof(SVgroupInfo));
for (const auto& vgs : it->second) {
for (const auto& vg : vgs.second->vgs) {
if (0 == vgSet.count(vg.vgId)) {
taosArrayPush(*pVgList, &vg);
vgSet.insert(vg.vgId);
}
}
}
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetAllDBVgList(SArray** pVgList) const {
std::set<int32_t> vgSet;
*pVgList = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVgroupInfo));
for (const auto& db : meta_) {
for (const auto& vgs : db.second) {
for (const auto& vg : vgs.second->vgs) {
if (0 == vgSet.count(vg.vgId)) {
taosArrayPush(*pVgList, &vg);
vgSet.insert(vg.vgId);
}
}
}
}
return TSDB_CODE_SUCCESS;
}
int32_t getAllDbCfg(SArray* pDbCfgReq, SArray** pDbCfgData) const {
int32_t code = TSDB_CODE_SUCCESS;
if (NULL != pDbCfgReq) {
@ -634,30 +659,29 @@ MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) {
MockCatalogService::~MockCatalogService() {}
ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, const std::string& tbname,
int8_t tableType, int32_t numOfColumns, int32_t numOfTags) {
ITableBuilder& MockCatalogService::createTableBuilder(const string& db, const string& tbname, int8_t tableType,
int32_t numOfColumns, int32_t numOfTags) {
return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags);
}
void MockCatalogService::createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname,
int16_t vgid) {
void MockCatalogService::createSubTable(const string& db, const string& stbname, const string& tbname, int16_t vgid) {
impl_->createSubTable(db, stbname, tbname, vgid);
}
void MockCatalogService::showTables() const { impl_->showTables(); }
void MockCatalogService::createFunction(const std::string& func, int8_t funcType, int8_t outputType, int32_t outputLen,
void MockCatalogService::createFunction(const string& func, int8_t funcType, int8_t outputType, int32_t outputLen,
int32_t bufSize) {
impl_->createFunction(func, funcType, outputType, outputLen, bufSize);
}
void MockCatalogService::createSmaIndex(const SMCreateSmaReq* pReq) { impl_->createSmaIndex(pReq); }
void MockCatalogService::createDnode(int32_t dnodeId, const std::string& host, int16_t port) {
void MockCatalogService::createDnode(int32_t dnodeId, const string& host, int16_t port) {
impl_->createDnode(dnodeId, host, port);
}
void MockCatalogService::createDatabase(const std::string& db, bool rollup, int8_t cacheLast) {
void MockCatalogService::createDatabase(const string& db, bool rollup, int8_t cacheLast) {
impl_->createDatabase(db, rollup, cacheLast);
}
@ -683,7 +707,7 @@ int32_t MockCatalogService::catalogGetDBCfg(const char* pDbFName, SDbCfgInfo* pD
return impl_->catalogGetDBCfg(pDbFName, pDbCfg);
}
int32_t MockCatalogService::catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const {
int32_t MockCatalogService::catalogGetUdfInfo(const string& funcName, SFuncInfo* pInfo) const {
return impl_->catalogGetUdfInfo(funcName, pInfo);
}

5
source/libs/planner/src/planLogicCreater.c
View File

@ -206,10 +206,9 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols
if (NULL == pScanPseudoCols) {
return SCAN_TYPE_TABLE;
}
int32_t funcType = ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType;
return FUNCTION_TYPE_BLOCK_DIST_INFO == funcType
return FUNCTION_TYPE_BLOCK_DIST_INFO == ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType
? SCAN_TYPE_BLOCK_INFO
: (FUNCTION_TYPE_TABLE_COUNT == funcType ? SCAN_TYPE_TABLE_COUNT : SCAN_TYPE_TABLE);
: SCAN_TYPE_TABLE;
}
return SCAN_TYPE_TABLE;

200
source/libs/planner/src/planOptimizer.c
View File

@ -16,6 +16,7 @@
#include "filter.h"
#include "functionMgt.h"
#include "planInt.h"
#include "systable.h"
#include "tglobal.h"
#include "ttime.h"
@ -64,6 +65,7 @@ typedef enum ECondAction {
} ECondAction;
typedef bool (*FMayBeOptimized)(SLogicNode* pNode);
typedef bool (*FShouldBeOptimized)(SLogicNode* pNode, void* pInfo);
static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) {
if (func(pNode)) {
@ -79,6 +81,19 @@ static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func)
return NULL;
}
static bool optFindEligibleNode(SLogicNode* pNode, FShouldBeOptimized func, void* pInfo) {
if (func(pNode, pInfo)) {
return true;
}
SNode* pChild;
FOREACH(pChild, pNode->pChildren) {
if (optFindEligibleNode((SLogicNode*)pChild, func, pInfo)) {
return true;
}
}
return false;
}
static void optResetParent(SLogicNode* pNode) {
SNode* pChild = NULL;
FOREACH(pChild, pNode->pChildren) { ((SLogicNode*)pChild)->pParent = pNode; }
@ -2440,6 +2455,188 @@ static int32_t pushDownLimitOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog
return TSDB_CODE_SUCCESS;
}
typedef struct STbCntScanOptInfo {
SAggLogicNode* pAgg;
SScanLogicNode* pScan;
SName table;
} STbCntScanOptInfo;
static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) {
if (1 != LIST_LENGTH(pAgg->pAggFuncs) || NULL != pAgg->pGroupKeys) {
return false;
}
SFunctionNode* pFunc = (SFunctionNode*)nodesListGetNode(pAgg->pAggFuncs, 0);
if (FUNCTION_TYPE_COUNT != pFunc->funcType) {
return false;
}
return true;
}
static bool tbCntScanOptGetColValFromCond(SOperatorNode* pOper, SColumnNode** pCol, SValueNode** pVal) {
if (OP_TYPE_EQUAL != pOper->opType) {
return false;
}
*pCol = NULL;
*pVal = NULL;
if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) {
*pCol = (SColumnNode*)pOper->pLeft;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) {
*pVal = (SValueNode*)pOper->pLeft;
}
if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) {
*pCol = (SColumnNode*)pOper->pRight;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) {
*pVal = (SValueNode*)pOper->pRight;
}
return NULL != *pCol && NULL != *pVal;
}
static bool tbCntScanOptIsEligibleLogicCond(STbCntScanOptInfo* pInfo, SLogicConditionNode* pCond) {
if (LOGIC_COND_TYPE_AND != pCond->condType) {
return false;
}
bool hasDbCond = false;
bool hasStbCond = false;
SColumnNode* pCol = NULL;
SValueNode* pVal = NULL;
SNode* pNode = NULL;
FOREACH(pNode, pCond->pParameterList) {
if (QUERY_NODE_OPERATOR != nodeType(pNode) || !tbCntScanOptGetColValFromCond((SOperatorNode*)pNode, &pCol, &pVal)) {
return false;
}
if (!hasDbCond && 0 == strcmp(pCol->colName, "db_name")) {
hasDbCond = true;
strcpy(pInfo->table.dbname, pVal->literal);
} else if (!hasStbCond && 0 == strcmp(pCol->colName, "stable_name")) {
hasStbCond = true;
strcpy(pInfo->table.tname, pVal->literal);
} else {
return false;
}
}
return hasDbCond;
}
static bool tbCntScanOptIsEligibleOpCond(SOperatorNode* pCond) {
SColumnNode* pCol = NULL;
SValueNode* pVal = NULL;
if (!tbCntScanOptGetColValFromCond(pCond, &pCol, &pVal)) {
return false;
}
return 0 == strcmp(pCol->colName, "db_name");
}
static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pConditions) {
if (NULL == pConditions) {
return true;
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pConditions)) {
return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions);
}
if (QUERY_NODE_OPERATOR == nodeType(pConditions)) {
return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions);
}
return false;
}
static bool tbCntScanOptIsEligibleScan(STbCntScanOptInfo* pInfo) {
if (0 != strcmp(pInfo->pScan->tableName.dbname, TSDB_INFORMATION_SCHEMA_DB) ||
0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_TABLE_TABLES) || NULL != pInfo->pScan->pGroupTags) {
return false;
}
return tbCntScanOptIsEligibleConds(pInfo, pInfo->pScan->node.pConditions);
}
static bool tbCntScanOptShouldBeOptimized(SLogicNode* pNode, STbCntScanOptInfo* pInfo) {
if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) ||
QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0))) {
return false;
}
pInfo->pAgg = (SAggLogicNode*)pNode;
pInfo->pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0);
return tbCntScanOptIsEligibleAgg(pInfo->pAgg) && tbCntScanOptIsEligibleScan(pInfo);
}
static SNode* tbCntScanOptCreateTableCountFunc() {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) {
return NULL;
}
strcpy(pFunc->functionName, "_table_count");
strcpy(pFunc->node.aliasName, "_table_count");
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc, NULL, 0)) {
nodesDestroyNode((SNode*)pFunc);
return NULL;
}
return (SNode*)pFunc;
}
static int32_t tbCntScanOptRewriteScan(STbCntScanOptInfo* pInfo, SScanLogicNode* pScan) {
pScan->scanType = SCAN_TYPE_TABLE_COUNT;
strcpy(pScan->tableName.dbname, pInfo->table.dbname);
strcpy(pScan->tableName.tname, pInfo->table.tname);
NODES_DESTORY_LIST(pScan->node.pTargets);
NODES_DESTORY_NODE(pScan->node.pConditions);
NODES_DESTORY_LIST(pScan->pScanCols);
NODES_DESTORY_LIST(pScan->pScanPseudoCols);
int32_t code = nodesListMakeStrictAppend(&pScan->pScanPseudoCols, tbCntScanOptCreateTableCountFunc());
if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExpr(nodesListGetNode(pScan->pScanPseudoCols, 0), &pScan->node.pTargets);
}
return code;
}
static int32_t tbCntScanOptCreateSumFunc(SFunctionNode* pCntFunc, SNode* pParam, SNode** pOutput) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(pFunc->functionName, "sum");
strcpy(pFunc->node.aliasName, pCntFunc->node.aliasName);
int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pParam));
if (TSDB_CODE_SUCCESS == code) {
code = fmGetFuncInfo(pFunc, NULL, 0);
}
if (TSDB_CODE_SUCCESS == code) {
*pOutput = (SNode*)pFunc;
} else {
nodesDestroyNode((SNode*)pFunc);
}
return code;
}
static int32_t tbCntScanOptRewriteAgg(SAggLogicNode* pAgg) {
SNode* pSum = NULL;
int32_t code = tbCntScanOptCreateSumFunc(
(SFunctionNode*)nodesListGetNode(pAgg->pAggFuncs, 0),
nodesListGetNode(((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))->pTargets, 0), &pSum);
if (TSDB_CODE_SUCCESS == code) {
NODES_DESTORY_LIST(pAgg->pAggFuncs);
code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, pSum);
}
return code;
}
static int32_t tableCountScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
STbCntScanOptInfo info = {0};
if (!optFindEligibleNode(pLogicSubplan->pNode, (FShouldBeOptimized)tbCntScanOptShouldBeOptimized, &info)) {
return TSDB_CODE_SUCCESS;
}
int32_t code = tbCntScanOptRewriteScan(&info, info.pScan);
if (TSDB_CODE_SUCCESS == code) {
code = tbCntScanOptRewriteAgg(info.pAgg);
}
return code;
}
// clang-format off
static const SOptimizeRule optimizeRuleSet[] = {
{.pName = "ScanPath", .optimizeFunc = scanPathOptimize},
@ -2454,7 +2651,8 @@ static const SOptimizeRule optimizeRuleSet[] = {
{.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize},
{.pName = "LastRowScan", .optimizeFunc = lastRowScanOptimize},
{.pName = "TagScan", .optimizeFunc = tagScanOptimize},
{.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize}
{.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize},
{.pName = "TableCountScan", .optimizeFunc = tableCountScanOptimize},
};
// clang-format on

1
source/libs/planner/src/planPhysiCreater.c
View File

@ -591,7 +591,6 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
pScan->accountId = pCxt->pPlanCxt->acctId;
pScan->sysInfo = pCxt->pPlanCxt->sysInfo;
if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLES) ||
0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLE_DISTRIBUTED) ||
0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS)) {
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
} else {

15
source/libs/planner/test/planSysTbTest.cpp
View File

@ -20,13 +20,6 @@ using namespace std;
class PlanSysTableTest : public PlannerTestBase {};
TEST_F(PlanSysTableTest, show) {
useDb("root", "test");
run("show tables");
run("show stables");
}
TEST_F(PlanSysTableTest, informationSchema) {
useDb("root", "information_schema");
@ -42,5 +35,13 @@ TEST_F(PlanSysTableTest, withAgg) {
TEST_F(PlanSysTableTest, tableCount) {
useDb("root", "information_schema");
run("SELECT COUNT(*) FROM ins_tables");
run("SELECT COUNT(*) FROM ins_tables WHERE db_name = 'test'");
run("SELECT COUNT(*) FROM ins_tables WHERE db_name = 'test' AND stable_name = 'st1'");
run("SELECT db_name, COUNT(*) FROM ins_tables GROUP BY db_name");
run("SELECT db_name, stable_name, COUNT(*) FROM ins_tables GROUP BY db_name, stable_name");
}