From 4c6dad1f43b0806ebbb3debc76c3a8d859d6fbcd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Nov 2021 13:21:54 +0800 Subject: [PATCH 01/16] [td-10564]Fix bug in parser. --- include/libs/parser/parser.h | 5 +- source/libs/parser/src/astValidate.c | 152 +++++++----------------- source/libs/parser/src/queryInfoUtil.c | 23 ++-- source/libs/parser/test/parserTests.cpp | 10 ++ source/libs/planner/src/planner.c | 34 +----- 5 files changed, 74 insertions(+), 150 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 9e6a033f10..2dec203ce8 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -108,10 +108,10 @@ typedef struct SQueryStmtInfo { SArray *pUdfInfo; struct SQueryStmtInfo *sibling; // sibling - SArray *pUpstream; // SArray struct SQueryStmtInfo *pDownstream; + SMultiFunctionsDesc info; + SArray *pUpstream; // SArray int32_t havingFieldNum; - SMultiFunctionsDesc info; int32_t exprListLevelIndex; } SQueryStmtInfo; @@ -176,6 +176,7 @@ typedef struct SSourceParam { SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize); int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); +int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo); STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex); SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 8e0f59b07e..a8a1c191f7 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -35,6 +35,11 @@ #define COLUMN_INDEX_INITIAL_VAL (-2) #define COLUMN_INDEX_INITIALIZER { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } +static int32_t resColId = 5000; +int32_t getNewResColId() { + return resColId++; +} + static int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, bool outerQuery, SMsgBuf* pMsgBuf); static int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId, STableMetaInfo** pTableMetaInfo, SSchema* columnSchema, tExprNode** pNode, SColumnIndex* pIndex, tSqlExprItem* pParamElem, SMsgBuf* pMsgBuf); @@ -1562,6 +1567,15 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* } } + for(int32_t i = 0; i < getExprFunctionLevel(pQueryInfo); ++i) { + SArray* functionList = extractFunctionList(pQueryInfo->exprList[i]); + extractFunctionDesc(functionList, &pQueryInfo->info); + + if ((code = checkForInvalidExpr(pQueryInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) { + return code; + } + } + return TSDB_CODE_SUCCESS; // Does not build query message here } @@ -1603,7 +1617,11 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { size_t size = getNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { SExprInfo* pExpr = getExprInfo(pQueryInfo, i); - int32_t functionId = getExprFunctionId(pExpr); + if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) { + continue; + } + + int32_t functionId = getExprFunctionId(pExpr); if (functionId == FUNCTION_COUNT && TSDB_COL_IS_TAG(pExpr->base.pColumns->flag)) { return buildInvalidOperationMsg(pMsgBuf, msg1); } @@ -1669,11 +1687,6 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { } } -static int32_t resColId = 5000; -int32_t getNewResColId() { - return resColId++; -} - int32_t addResColumnInfo(SQueryStmtInfo* pQueryInfo, int32_t outputIndex, SSchema* pSchema, SExprInfo* pSqlExpr) { SInternalField* pInfo = insertFieldInfo(&pQueryInfo->fieldsInfo, outputIndex, pSchema); pInfo->pExpr = pSqlExpr; @@ -1787,7 +1800,6 @@ static int32_t checkForAliasName(SMsgBuf* pMsgBuf, char* aliasName) { return TSDB_CODE_SUCCESS; } -static int32_t validateComplexExpr(tSqlExpr* pExpr, SQueryStmtInfo* pQueryInfo, SArray* pColList, int32_t* type, SMsgBuf* pMsgBuf); static int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStmtInfo* pQueryInfo, SArray* pCols, bool* keepTableCols, SMsgBuf* pMsgBuf); static int64_t getTickPerSecond(SVariant* pVariant, int32_t precision, int64_t* tickPerSec, SMsgBuf *pMsgBuf) { @@ -2600,15 +2612,15 @@ static int32_t validateExprLeafFunctionNode(SQueryStmtInfo* pQueryInfo, tSqlExpr return TSDB_CODE_SUCCESS; } -int32_t validateScalarFunctionParamNum(tSqlExprItem* pItem, SMsgBuf* pMsgBuf) { +static int32_t validateScalarFunctionParamNum(tSqlExpr* pSqlExpr, int32_t functionId, SMsgBuf* pMsgBuf) { int32_t code = TSDB_CODE_SUCCESS; - switch (pItem->functionId) { + switch (functionId) { case FUNCTION_CEIL: { - code = checkForkParam(pItem->pNode, 1, pMsgBuf); + code = checkForkParam(pSqlExpr, 1, pMsgBuf); break; } case FUNCTION_LENGTH: { - code = checkForkParam(pItem->pNode, 1, pMsgBuf); + code = checkForkParam(pSqlExpr, 1, pMsgBuf); break; } } @@ -2616,38 +2628,6 @@ int32_t validateScalarFunctionParamNum(tSqlExprItem* pItem, SMsgBuf* pMsgBuf) { return code; } -int32_t validateScalarFunctionParam(SQueryStmtInfo* pQueryInfo, tSqlExpr* pExpr, SArray* pList, int32_t* exprType, SMsgBuf* pMsgBuf) { - int32_t code = TSDB_CODE_SUCCESS; - - SArray* pParamList = pExpr->Expr.paramList; - *exprType = NORMAL_ARITHMETIC; - - for (int32_t i = 0; i < 1; ++i) { - tSqlExprItem* pParamElem = taosArrayGet(pParamList, i); - tSqlExpr* pSqlExpr = pParamElem->pNode; - - int32_t type = pSqlExpr->type; - if (type == SQL_NODE_VALUE) { - // do nothing for scalar function, or maybe the evaluation can be done here - } else if (type == SQL_NODE_SQLFUNCTION) { - code = validateExprLeafFunctionNode(pQueryInfo, pSqlExpr, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else if (type == SQL_NODE_EXPR) { - code = validateComplexExpr(pSqlExpr, pQueryInfo, pList, exprType, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else if (type == SQL_NODE_TABLE_COLUMN) { - code = validateExprLeafColumnNode(pQueryInfo, &pSqlExpr->columnName, pList, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } -} - SExprInfo* doAddProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, SColumnIndex* pColIndex, const char* aliasName, int32_t colId) { STableMeta* pTableMeta = getMetaInfo(pQueryInfo, pColIndex->tableIndex)->pTableMeta; @@ -2888,57 +2868,6 @@ static int32_t validateExprLeafNode(tSqlExpr* pExpr, SQueryStmtInfo* pQueryInfo, return TSDB_CODE_SUCCESS; } -int32_t validateComplexExpr(tSqlExpr * pExpr, SQueryStmtInfo* pQueryInfo, SArray* pColList, int32_t* type, SMsgBuf* pMsgBuf) { - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - int32_t code = TSDB_CODE_SUCCESS; - if (pExpr->type == SQL_NODE_SQLFUNCTION) { - return validateScalarFunctionParam(pQueryInfo, pExpr, pColList, type, pMsgBuf); - } - - tSqlExpr* pLeft = pExpr->pLeft; - if (pLeft->type == SQL_NODE_EXPR) { - code = validateComplexExpr(pLeft, pQueryInfo, pColList, type, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - code = validateExprLeafNode(pLeft, pQueryInfo, pColList, type, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - tSqlExpr* pRight = pExpr->pRight; - if (pRight->type == SQL_NODE_EXPR) { - code = validateComplexExpr(pRight, pQueryInfo, pColList, type, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - code = validateExprLeafNode(pRight, pQueryInfo, pColList, type, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - // check divide by 0 - if (pExpr->tokenId == TK_DIVIDE && pRight->type == SQL_NODE_VALUE) { - int32_t type1 = pRight->value.nType; - const char* msg1 = "invalid expr (divide by 0)"; - - if (type1 == TSDB_DATA_TYPE_DOUBLE && pRight->value.d < DBL_EPSILON) { - return buildInvalidOperationMsg(pMsgBuf, msg1); - } else if (type1 == TSDB_DATA_TYPE_INT && pRight->value.i == 0) { - return buildInvalidOperationMsg(pMsgBuf, msg1); - } - } - - return TSDB_CODE_SUCCESS; -} - static uint64_t findTmpSourceColumnInNextLevel(SQueryStmtInfo* pQueryInfo, tExprNode *pExpr) { // This function must be a aggregate function, so it must be in the next level pQueryInfo->exprListLevelIndex += 1; @@ -3080,7 +3009,8 @@ int32_t validateSqlExpr(const tSqlExpr* pSqlExpr, SQueryStmtInfo *pQueryInfo, SM } int32_t tokenId = pSqlExpr->tokenId; - if (pRight->type == SQL_NODE_VALUE && pRight->value.nType == TSDB_DATA_TYPE_DOUBLE && pRight->value.d == 0 && tokenId == TK_DIVIDE) { + if (pRight->type == SQL_NODE_VALUE && (pRight->value.nType == TSDB_DATA_TYPE_DOUBLE || pRight->value.nType == TSDB_DATA_TYPE_INT) && + pRight->value.d == 0 && tokenId == TK_DIVIDE) { return buildInvalidOperationMsg(pMsgBuf, "invalid expression (divided by 0)"); } @@ -3098,6 +3028,20 @@ int32_t validateSqlExpr(const tSqlExpr* pSqlExpr, SQueryStmtInfo *pQueryInfo, SM if (ret != TSDB_CODE_SUCCESS) { return ret; } + } else if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { + bool scalar = false; + int32_t functionId = qIsBuiltinFunction(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n, &scalar); + if (functionId < 0) { + return buildInvalidOperationMsg(pMsgBuf, "invalid function name"); + } + + // do check the parameter number for scalar function + if (scalar) { + int32_t ret = validateScalarFunctionParamNum(pSqlExpr, functionId, pMsgBuf); + if (ret != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, "invalid number of function parameters"); + } + } } return TSDB_CODE_SUCCESS; @@ -3379,14 +3323,14 @@ int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, if (type == SQL_NODE_SQLFUNCTION) { bool scalarFunc = false; pItem->functionId = qIsBuiltinFunction(pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n, &scalarFunc); - if (pItem->functionId == FUNCTION_INVALID_ID) { - int32_t functionId = FUNCTION_INVALID_ID; - bool valid = qIsValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n, &functionId); - if (!valid) { + if (pItem->functionId == FUNCTION_INVALID_ID) { // temporarily disable the udf +// int32_t functionId = FUNCTION_INVALID_ID; +// bool valid = qIsValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n, &functionId); +// if (!valid) { return buildInvalidOperationMsg(pMsgBuf, msg5); - } +// } - pItem->functionId = functionId; +// pItem->functionId = functionId; } if (scalarFunc) { // scalar function @@ -3920,12 +3864,6 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer validateSqlNode(p, pQueryInfo, &buf); } - SArray* functionList = extractFunctionList(pQueryInfo->exprList[0]); - extractFunctionDesc(functionList, &pQueryInfo->info); - - if ((code = checkForInvalidExpr(pQueryInfo, &buf)) != TSDB_CODE_SUCCESS) { - return code; - } return code; } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 04922ce5ad..0543478674 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -223,7 +223,7 @@ void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t byt } int32_t getExprFunctionId(SExprInfo *pExprInfo) { - assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_UNARYEXPR_NODE); + assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_FUNCTION_NODE); return 0; } @@ -350,11 +350,16 @@ bool tscHasColumnFilter(SQueryStmtInfo* pQueryInfo) { return false; } -//void tscClearInterpInfo(SQueryStmtInfo* pQueryInfo) { -// if (!tscIsPointInterpQuery(pQueryInfo)) { -// return; -// } -// -// pQueryInfo->fillType = TSDB_FILL_NONE; -// tfree(pQueryInfo->fillVal); -//} \ No newline at end of file +int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo) { + int32_t n = 10; + + int32_t level = 0; + for(int32_t i = 0; i < n; ++i) { + SArray* pList = pQueryInfo->exprList[i]; + if (taosArrayGetSize(pList) > 0) { + level += 1; + } + } + + return level; +} \ No newline at end of file diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 6e491cedb0..1ecd9e3a08 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -412,6 +412,16 @@ TEST(testCase, function_Test10) { sqlCheck("select block_dist() from `t.1abc`", true); sqlCheck("select block_dist(a) from `t.1abc`", false); sqlCheck("select count(*) from `t.1abc` interval(1s) group by a", false); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + sqlCheck("select length119(a,b) from `t.1abc`", false); + sqlCheck("select length(a,b) from `t.1abc`", false); + sqlCheck("select block_dist() + 20 from `t.1abc`", false); + sqlCheck("select top(a, 20), count(b) from `t.1abc`", false); + sqlCheck("select count(b), c from `t.1abc`", false); + sqlCheck("select last_row(*), count(b) from `t.1abc`", false); + sqlCheck("select last_row(a, b) + 20 from `t.1abc`", false); + sqlCheck("select last_row(count(*)) from `t.1abc`", false); } TEST(testCase, function_Test6) { diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index e70b4a1280..804fe5c3bc 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -217,42 +217,12 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe return pNode; } -static int32_t getFunctionLevel(SQueryStmtInfo* pQueryInfo) { - int32_t n = 10; - - int32_t level = 0; - for(int32_t i = 0; i < n; ++i) { - SArray* pList = pQueryInfo->exprList[i]; - if (taosArrayGetSize(pList) > 0) { - level += 1; - } - } - - return level; -} - -static SQueryPlanNode* createOneQueryPlanNode(SArray* p, SQueryPlanNode* pNode, SExprInfo* pExpr, SQueryTableInfo* info) { - if (pExpr->pExpr->nodeType == TEXPR_FUNCTION_NODE) { - bool aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName); - if (aggregateFunc) { - int32_t numOfOutput = (int32_t)taosArrayGetSize(p); - return createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, numOfOutput, info, NULL); - } else { - int32_t numOfOutput = (int32_t)taosArrayGetSize(p); - return createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, numOfOutput, info, NULL); - } - } else { - int32_t numOfOutput = (int32_t)taosArrayGetSize(p); - return createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, numOfOutput, info, NULL); - } -} - static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { // group by column not by tag size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); // check for aggregation - int32_t level = getFunctionLevel(pQueryInfo); + int32_t level = getExprFunctionLevel(pQueryInfo); for(int32_t i = level - 1; i >= 0; --i) { SArray* p = pQueryInfo->exprList[i]; @@ -344,7 +314,7 @@ static bool isAllAggExpr(SArray* pList) { static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo) { assert(pQueryInfo != NULL); - size_t level = getFunctionLevel(pQueryInfo); + size_t level = getExprFunctionLevel(pQueryInfo); for(int32_t i = 0; i < level - 1; ++i) { SArray* p = pQueryInfo->exprList[i]; From 8ba0780355413bff8e065c1c4af198226a175f57 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 14 Nov 2021 15:49:12 +0800 Subject: [PATCH 02/16] [td-10564] Fix bug in parse nest sql query. --- include/common/common.h | 1 + include/libs/planner/planner.h | 29 +++++++ source/libs/executor/inc/executorimpl.h | 28 ------- source/libs/function/src/tfunction.c | 19 +---- source/libs/parser/inc/astGenerator.h | 5 +- source/libs/parser/src/astValidate.c | 104 +++++++++++++++++------- source/libs/parser/src/queryInfoUtil.c | 8 +- source/libs/parser/test/parserTests.cpp | 18 ++-- source/libs/planner/inc/plannerInt.h | 58 +++++++++++-- source/libs/planner/src/planner.c | 53 ++++++------ 10 files changed, 206 insertions(+), 117 deletions(-) diff --git a/include/common/common.h b/include/common/common.h index 0913c12597..d99e4a78b7 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -58,6 +58,7 @@ typedef struct SDataBlockInfo { typedef struct SSDataBlock { SColumnDataAgg *pBlockAgg; SArray *pDataBlock; // SArray + SArray *pTagsList; // SArray for tag value SDataBlockInfo info; } SSDataBlock; diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 3ccc4bf4cd..1ff3f02da5 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -23,6 +23,35 @@ extern "C" { #define QUERY_TYPE_MERGE 1 #define QUERY_TYPE_PARTIAL 2 +enum OPERATOR_TYPE_E { + OP_TableScan = 1, + OP_DataBlocksOptScan = 2, + OP_TableSeqScan = 3, + OP_TagScan = 4, + OP_TableBlockInfoScan= 5, + OP_Aggregate = 6, + OP_Project = 7, + OP_Groupby = 8, + OP_Limit = 9, + OP_SLimit = 10, + OP_TimeWindow = 11, + OP_SessionWindow = 12, + OP_StateWindow = 22, + OP_Fill = 13, + OP_MultiTableAggregate = 14, + OP_MultiTableTimeInterval = 15, +// OP_DummyInput = 16, //TODO remove it after fully refactor. +// OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream. +// OP_GlobalAggregate = 18, // global merge for the multi-way data sources. + OP_Filter = 19, + OP_Distinct = 20, + OP_Join = 21, + OP_AllTimeWindow = 23, + OP_AllMultiTableTimeInterval = 24, + OP_Order = 25, + OP_Exchange = 26, +}; + struct SEpSet; struct SQueryPlanNode; struct SQueryDistPlanNode; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a99717a123..907fb4d2bf 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -279,34 +279,6 @@ enum { OP_EXEC_DONE = 3, }; -enum OPERATOR_TYPE_E { - OP_TableScan = 1, - OP_DataBlocksOptScan = 2, - OP_TableSeqScan = 3, - OP_TagScan = 4, - OP_TableBlockInfoScan= 5, - OP_Aggregate = 6, - OP_Project = 7, - OP_Groupby = 8, - OP_Limit = 9, - OP_SLimit = 10, - OP_TimeWindow = 11, - OP_SessionWindow = 12, - OP_Fill = 13, - OP_MultiTableAggregate = 14, - OP_MultiTableTimeInterval = 15, - OP_DummyInput = 16, //TODO remove it after fully refactor. - OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream. - OP_GlobalAggregate = 18, // global merge for the multi-way data sources. - OP_Filter = 19, - OP_Distinct = 20, - OP_Join = 21, - OP_StateWindow = 22, - OP_AllTimeWindow = 23, - OP_AllMultiTableTimeInterval = 24, - OP_Order = 25, -}; - typedef struct SOperatorInfo { uint8_t operatorType; bool blockingOptr; // block operator or not diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c index d3fc19a47f..7e3b4bcd5d 100644 --- a/source/libs/function/src/tfunction.c +++ b/source/libs/function/src/tfunction.c @@ -182,25 +182,14 @@ bool isArithmeticQueryOnAggResult(SArray* pFunctionIdList) { return false; } -bool isGroupbyColumn(SArray* pFunctionIdList) { -// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); -// int32_t numOfCols = getNumOfColumns(pTableMetaInfo->pTableMeta); -// -// SGroupbyExpr* pGroupbyExpr = &pQueryInfo->groupbyExpr; -// for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { -// SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); -// if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < numOfCols) { // group by normal columns -// return true; -// } -// } - - return false; +bool isGroupbyColumn(SGroupbyExpr* pGroupby) { + return !pGroupby->groupbyTag; } bool isTopBotQuery(SArray* pFunctionIdList) { int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); for (int32_t i = 0; i < num; ++i) { - int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i); + char* f = *(char**) taosArrayGet(pFunctionIdList, i); if (f == FUNCTION_TS) { continue; } @@ -432,7 +421,6 @@ bool hasTagValOutput(SArray* pFunctionIdList) { void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc) { assert(pFunctionIdList != NULL); - pDesc->blockDistribution = isBlockDistQuery(pFunctionIdList); if (pDesc->blockDistribution) { return; @@ -441,4 +429,5 @@ void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc) { pDesc->projectionQuery = isProjectionQuery(pFunctionIdList); pDesc->onlyTagQuery = isTagsQuery(pFunctionIdList); pDesc->interpQuery = isInterpQuery(pFunctionIdList); + pDesc->topbotQuery = isTopBotQuery(pFunctionIdList); } diff --git a/source/libs/parser/inc/astGenerator.h b/source/libs/parser/inc/astGenerator.h index 863c307f34..cb3d459de6 100644 --- a/source/libs/parser/inc/astGenerator.h +++ b/source/libs/parser/inc/astGenerator.h @@ -294,7 +294,10 @@ SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SSqlNode *pSe SAlterTableInfo * tSetAlterTableInfo(SToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, int16_t tableType); SCreatedTableInfo createNewChildTableInfo(SToken *pTableName, SArray *pTagNames, SArray *pTagVals, SToken *pToken, SToken *igExists); - +/*! + * test + * @param pSqlNode + */ void destroyAllSqlNode(struct SSubclause *pSqlNode); void destroySqlNode(SSqlNode *pSql); void freeCreateTableInfo(void* p); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index a8a1c191f7..2536f8f060 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -1607,6 +1607,30 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { if (pQueryInfo->fillType != TSDB_FILL_NONE) { return buildInvalidOperationMsg(pMsgBuf, msg4); } + + // select top(col, k), count(*) from table_name + int32_t num = 0; + SExprInfo* pMain = NULL; + size_t size = getNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { + SExprInfo* pExpr = getExprInfo(pQueryInfo, i); + const char* functionName = pExpr->pExpr->_function.functionName; + + if (strcmp(functionName, "top") != 0 && strcmp(functionName, "bottom") != 0) { + if (qIsAggregateFunction(functionName)) { + return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select clause"); + } + + // the primary key is valid + if (pExpr->pExpr->nodeType == TEXPR_COL_NODE) { + if (pExpr->pExpr->pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + continue; + } + } + + continue; + } + } } /* @@ -2628,35 +2652,51 @@ static int32_t validateScalarFunctionParamNum(tSqlExpr* pSqlExpr, int32_t functi return code; } -SExprInfo* doAddProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, SColumnIndex* pColIndex, const char* aliasName, int32_t colId) { +int32_t doAddProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, SColumnIndex* pColIndex, + const char* aliasName, int32_t colId, SMsgBuf* pMsgBuf) { STableMeta* pTableMeta = getMetaInfo(pQueryInfo, pColIndex->tableIndex)->pTableMeta; SSchema* pSchema = getOneColumnSchema(pTableMeta, pColIndex->columnIndex); - SColumnIndex index = *pColIndex; - - char* funcName = NULL; - if (TSDB_COL_IS_TAG(index.type)) { - int32_t numOfCols = getNumOfColumns(pTableMeta); - index.columnIndex = pColIndex->columnIndex - numOfCols; - funcName = "project_tag"; - } else { - index.columnIndex = pColIndex->columnIndex; - funcName = "project_col"; - } const char* name = (aliasName == NULL)? pSchema->name:aliasName; SSchema s = createSchema(pSchema->type, pSchema->bytes, colId, name); - STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); - SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, pSchema); + tExprNode *pNode = NULL; + bool keepTableCols = true; + SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); - SSourceParam param = {0}; - addIntoSourceParam(¶m, NULL, &c); + tSqlExpr sqlNode = {0}; + sqlNode.type = SQL_NODE_TABLE_COLUMN; - return doAddOneExprInfo(pQueryInfo, funcName, ¶m, outputColIndex, pTableMetaInfo, &s, 0, s.name, true); + SToken colNameToken = {.z = pSchema->name, .n = strlen(pSchema->name)}; + sqlNode.columnName = colNameToken; + + int32_t ret = sqlExprToExprNode(&pNode, &sqlNode, pQueryInfo, pColumnList, &keepTableCols, pMsgBuf); + if (ret != TSDB_CODE_SUCCESS) { + tExprTreeDestroy(pNode, NULL); + return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select clause"); + } + + SExprInfo* pExpr = createBinaryExprInfo(pNode, &s); + tstrncpy(pExpr->base.resSchema.name, name, tListLen(pExpr->base.resSchema.name)); + tstrncpy(pExpr->base.token, name, tListLen(pExpr->base.token)); + + SArray* pExprList = getCurrentExprList(pQueryInfo); + addExprInfo(pExprList, outputColIndex, pExpr, pQueryInfo->exprListLevelIndex); + + // extract columns according to the tExprNode tree + size_t num = taosArrayGetSize(pColumnList); + pExpr->base.pColumns = calloc(num, sizeof(SColumn)); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGet(pColumnList, i); + pExpr->base.pColumns[i] = *pCol; + } + + pExpr->base.numOfCols = num; + return TSDB_CODE_SUCCESS; } -static int32_t doAddProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos) { +static int32_t doAddProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos, SMsgBuf* pMsgBuf) { STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -2669,7 +2709,7 @@ static int32_t doAddProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, SColu for (int32_t j = 0; j < numOfTotalColumns; ++j) { pIndex->columnIndex = j; - doAddProjectCol(pQueryInfo, startPos + j, pIndex, NULL, getNewResColId()); + doAddProjectCol(pQueryInfo, startPos + j, pIndex, NULL, getNewResColId(), pMsgBuf); } return numOfTotalColumns; @@ -2778,11 +2818,11 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* if (index.tableIndex == COLUMN_INDEX_INITIAL_VAL) { // all table columns are required. for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { index.tableIndex = i; - int32_t inc = doAddProjectionExprAndResColumn(pQueryInfo, &index, startPos); + int32_t inc = doAddProjectionExprAndResColumn(pQueryInfo, &index, startPos, pMsgBuf); startPos += inc; } } else { - doAddProjectionExprAndResColumn(pQueryInfo, &index, startPos); + doAddProjectionExprAndResColumn(pQueryInfo, &index, startPos, pMsgBuf); } // add the primary timestamp column even though it is not required by user @@ -2821,7 +2861,7 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* return buildInvalidOperationMsg(pMsgBuf, msg1); } - doAddProjectCol(pQueryInfo, startPos, &index, pItem->aliasName, getNewResColId()); + doAddProjectCol(pQueryInfo, startPos, &index, pItem->aliasName, getNewResColId(), pMsgBuf); } // add the primary timestamp column even though it is not required by user @@ -2920,12 +2960,12 @@ static tExprNode* doCreateColumnNode(SQueryStmtInfo* pQueryInfo, SColumnIndex* p static int32_t validateSqlExpr(const tSqlExpr* pSqlExpr, SQueryStmtInfo *pQueryInfo, SMsgBuf* pMsgBuf); -static int32_t doProcessFunctionLeafNodeParam(SQueryStmtInfo* pQueryInfo, int32_t* num, tExprNode** p, SArray* pCols, +static int32_t doProcessFunctionLeafNodeParam(SQueryStmtInfo* pQueryInfo, int32_t* num, tExprNode*** p, SArray* pCols, bool* keepTableCols, const tSqlExpr* pSqlExpr, SMsgBuf* pMsgBuf) { SArray* pParamList = pSqlExpr->Expr.paramList; if (pParamList != NULL) { *num = taosArrayGetSize(pParamList); - p = calloc((*num), POINTER_BYTES); + (*p) = calloc((*num), POINTER_BYTES); for (int32_t i = 0; i < (*num); ++i) { tSqlExprItem* pItem = taosArrayGet(pParamList, i); @@ -2935,7 +2975,7 @@ static int32_t doProcessFunctionLeafNodeParam(SQueryStmtInfo* pQueryInfo, int32_ return ret; } - int32_t code = sqlExprToExprNode(&p[i], pItem->pNode, pQueryInfo, pCols, keepTableCols, pMsgBuf); + int32_t code = sqlExprToExprNode(&(*p)[i], pItem->pNode, pQueryInfo, pCols, keepTableCols, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2946,10 +2986,10 @@ static int32_t doProcessFunctionLeafNodeParam(SQueryStmtInfo* pQueryInfo, int32_ } *num = 1; - p = calloc(*num, POINTER_BYTES); + (*p) = calloc(*num, POINTER_BYTES); SColumnIndex index = {.type = TSDB_COL_NORMAL, .tableIndex = 0, .columnIndex = 0}; - p[0] = doCreateColumnNode(pQueryInfo, &index, *keepTableCols, pCols); + (*p)[0] = doCreateColumnNode(pQueryInfo, &index, *keepTableCols, pCols); } return TSDB_CODE_SUCCESS; @@ -3037,7 +3077,7 @@ int32_t validateSqlExpr(const tSqlExpr* pSqlExpr, SQueryStmtInfo *pQueryInfo, SM // do check the parameter number for scalar function if (scalar) { - int32_t ret = validateScalarFunctionParamNum(pSqlExpr, functionId, pMsgBuf); + int32_t ret = validateScalarFunctionParamNum((tSqlExpr*) pSqlExpr, functionId, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { return buildInvalidOperationMsg(pMsgBuf, "invalid number of function parameters"); } @@ -3087,7 +3127,7 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStm int32_t num = 0; tExprNode** p = NULL; - int32_t code = doProcessFunctionLeafNodeParam(pQueryInfo, &num, p, pCols, keepTableCols, pSqlExpr, pMsgBuf); + int32_t code = doProcessFunctionLeafNodeParam(pQueryInfo, &num, &p, pCols, keepTableCols, pSqlExpr, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3147,6 +3187,9 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStm (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); strncpy((*pExpr)->pSchema->name, pSqlExpr->exprToken.z, pSqlExpr->exprToken.n); + // it must be the aggregate function + assert(qIsAggregateFunction((*pExpr)->pSchema->name)); + uint64_t uid = findTmpSourceColumnInNextLevel(pQueryInfo, *pExpr); if (!(*keepTableCols)) { SColumn c = createColumn(uid, NULL, TSDB_COL_TMP, (*pExpr)->pSchema); @@ -3345,7 +3388,8 @@ int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, } } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) { // use the dynamic array list to decide if the function is valid or not - // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 + // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 + // todo refacto to remove this function if ((code = addProjectionExprAndResColumn(pQueryInfo, pItem, outerQuery, pMsgBuf)) != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 0543478674..64156f3843 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -324,10 +324,14 @@ SArray* extractFunctionList(SArray* pExprInfoList) { assert(pExprInfoList != NULL); size_t len = taosArrayGetSize(pExprInfoList); - SArray* p = taosArrayInit(len, sizeof(int32_t)); + SArray* p = taosArrayInit(len, POINTER_BYTES); for(int32_t i = 0; i < len; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pExprInfoList, i); - taosArrayPush(p, &pExprInfo->pExpr->_function.functionName); + if (pExprInfo->pExpr->nodeType == TEXPR_FUNCTION_NODE) { + taosArrayPush(p, &pExprInfo->pExpr->_function.functionName); + } else { + taosArrayPush(p, ""); + } } return p; diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 1ecd9e3a08..5a240061b2 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -398,6 +398,7 @@ TEST(testCase, function_Test5) { TEST(testCase, function_Test10) { sqlCheck("select c from `t.1abc`", true); sqlCheck("select length(c) from `t.1abc`", true); + sqlCheck("select length(sum(col)) from `t.1abc`", true); sqlCheck("select sum(length(a+b)) from `t.1abc`", true); sqlCheck("select sum(sum(a+b)) from `t.1abc`", false); sqlCheck("select sum(length(a) + length(b)) from `t.1abc`", true); @@ -406,6 +407,8 @@ TEST(testCase, function_Test10) { sqlCheck("select cov(a, b) from `t.1abc`", true); sqlCheck("select sum(length(a) + count(b)) from `t.1abc`", false); + sqlCheck("select concat(sum(a), count(b)) from `t.1abc`", true); + sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); sqlCheck("select length(length(length(a))) from `t.1abc`", true); sqlCheck("select count() from `t.1abc`", false); @@ -415,13 +418,16 @@ TEST(testCase, function_Test10) { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// sqlCheck("select length119(a,b) from `t.1abc`", false); - sqlCheck("select length(a,b) from `t.1abc`", false); - sqlCheck("select block_dist() + 20 from `t.1abc`", false); - sqlCheck("select top(a, 20), count(b) from `t.1abc`", false); + sqlCheck("select length(a, b) from `t.1abc`", false); + sqlCheck("select block_dist() + 20 from `t.1abc`", true); sqlCheck("select count(b), c from `t.1abc`", false); - sqlCheck("select last_row(*), count(b) from `t.1abc`", false); - sqlCheck("select last_row(a, b) + 20 from `t.1abc`", false); - sqlCheck("select last_row(count(*)) from `t.1abc`", false); + sqlCheck("select top(a, 20), count(b) from `t.1abc`", false); +// sqlCheck("select top(a, 20), b from `t.1abc`", false); +// sqlCheck("select top(a, 20), a+20 from `t.1abc`", true); +// sqlCheck("select top(a, 20), bottom(a, 10) from `t.1abc`", false); +// sqlCheck("select last_row(*), count(b) from `t.1abc`", false); +// sqlCheck("select last_row(a, b) + 20 from `t.1abc`", false); +// sqlCheck("select last_row(count(*)) from `t.1abc`", false); } TEST(testCase, function_Test6) { diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 27a96b539e..6bd89905b1 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -26,18 +26,26 @@ extern "C" { #include "taosmsg.h" typedef struct SQueryNodeBasicInfo { - int32_t type; - char *name; + int32_t type; // operator type + char *name; // operator name } SQueryNodeBasicInfo; +typedef struct SQueryDistPlanNodeInfo { + bool stableQuery; // super table query or not + int32_t phase; // merge|partial + int32_t type; // operator type + char *name; // operator name + SEpSet *sourceEp; // data source epset +} SQueryDistPlanNodeInfo; + typedef struct SQueryTableInfo { - char *tableName; - uint64_t uid; + char *tableName; + uint64_t uid; + STimeWindow window; } SQueryTableInfo; typedef struct SQueryPlanNode { SQueryNodeBasicInfo info; - SQueryTableInfo tableInfo; SSchema *pSchema; // the schema of the input SSDatablock int32_t numOfCols; // number of input columns SArray *pExpr; // the query functions or sql aggregations @@ -51,9 +59,49 @@ typedef struct SQueryPlanNode { typedef struct SQueryDistPlanNode { SQueryNodeBasicInfo info; + SSchema *pSchema; // the schema of the input SSDatablock + int32_t numOfCols; // number of input columns + SArray *pExpr; // the query functions or sql aggregations + int32_t numOfExpr; // number of result columns, which is also the number of pExprs + void *pExtInfo; // additional information + // previous operator to generated result for current node to process + // in case of join, multiple prev nodes exist. + SArray *pPrevNodes; // upstream nodes, or exchange operator to load data from multiple sources. } SQueryDistPlanNode; +typedef struct SQueryCostSummary { + int64_t startTs; // Object created and added into the message queue + int64_t endTs; // the timestamp when the task is completed + int64_t cputime; // total cpu cost, not execute elapsed time + + int64_t loadRemoteDataDuration; // remote io time + int64_t loadNativeDataDuration; // native disk io time + + uint64_t loadNativeData; // blocks + SMA + header files + uint64_t loadRemoteData; // remote data acquired by exchange operator. + + uint64_t waitDuration; // the time to waiting to be scheduled in queue does matter, so we need to record it + int64_t addQTs; // the time to be added into the message queue, used to calculate the waiting duration in queue. + + uint64_t totalRows; + uint64_t loadRows; + uint32_t totalBlocks; + uint32_t loadBlocks; + uint32_t loadBlockAgg; + uint32_t skipBlocks; + uint64_t resultSize; // generated result size in Kb. +} SQueryCostSummary; + +typedef struct SQueryTask { + uint64_t queryId; // query id + uint64_t taskId; // task id + SQueryDistPlanNode *pNode; // operator tree + uint64_t status; // task status + SQueryCostSummary summary; // task execution summary + void *pOutputHandle; // result buffer handle, to temporarily keep the output result for next stage +} SQueryTask; + #ifdef __cplusplus } #endif diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 804fe5c3bc..101ea3ec2f 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -95,17 +95,12 @@ int32_t qCreateQueryJob(const struct SQueryDistPlanNode* pPhyNode, struct SQuery //====================================================================================================================== static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, - SExprInfo** pExpr, int32_t numOfOutput, SQueryTableInfo* pTableInfo, void* pExtInfo) { + SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) { SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode)); pNode->info.type = type; pNode->info.name = strdup(name); - if (pTableInfo->uid != 0 && pTableInfo->tableName) { // it is a true table - pNode->tableInfo.uid = pTableInfo->uid; - pNode->tableInfo.tableName = strdup(pTableInfo->tableName); - } - pNode->numOfExpr = numOfOutput; pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES); @@ -120,9 +115,10 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla switch(type) { case QNODE_TABLESCAN: { - STimeWindow* window = calloc(1, sizeof(STimeWindow)); - memcpy(window, pExtInfo, sizeof(STimeWindow)); - pNode->pExtInfo = window; + SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo)); + memcpy(info, pExtInfo, sizeof(SQueryTableInfo)); + info->tableName = strdup(((SQueryTableInfo*) pExtInfo)->tableName); + pNode->pExtInfo = info; break; } @@ -179,21 +175,20 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe SArray* pExprs, SArray* tableCols) { if (pQueryInfo->info.onlyTagQuery) { int32_t num = (int32_t) taosArrayGetSize(pExprs); - SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info, NULL); + SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, NULL); if (pQueryInfo->info.distinct) { - pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, info, NULL); + pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL); } return pNode; } - STimeWindow* window = &pQueryInfo->window; - SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info, window); + SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info); if (pQueryInfo->info.projectionQuery) { int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, info, NULL); + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, NULL); } else { STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0); @@ -210,7 +205,7 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe pExpr[i] = p; } - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, info, NULL); + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL); tfree(pExpr); } @@ -243,24 +238,24 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQuer if (aggregateFunc) { if (pQueryInfo->interval.interval > 0) { - pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, info, &pQueryInfo->interval); + pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->interval); } else if (pQueryInfo->sessionWindow.gap > 0) { - pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, info, &pQueryInfo->sessionWindow); + pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->sessionWindow); } else if (pQueryInfo->stateWindow.col.info.colId > 0) { - pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, info, &pQueryInfo->stateWindow); + pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->stateWindow); } else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) { - pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, info, &pQueryInfo->groupbyExpr); + pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, &pQueryInfo->groupbyExpr); } else { - pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, info, NULL); + pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, NULL); } } else { - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, info, NULL); + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL); } } if (pQueryInfo->havingFieldNum > 0) { // int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); -// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, NULL); +// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, NULL); } if (pQueryInfo->fillType != TSDB_FILL_NONE) { @@ -269,11 +264,11 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQuer pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t)); memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); - pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, NULL, 0, info, pInfo); + pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, NULL, 0, pInfo); } if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) { - pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, info, &pQueryInfo->limit); + pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit); } return pNode; @@ -399,7 +394,7 @@ SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { SQueryTableInfo info = {0}; int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]); SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, - pQueryInfo->exprList[0]->pData, num, &info, NULL); + pQueryInfo->exprList[0]->pData, num, NULL); // 4. add the aggregation or projection execution node pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); @@ -419,8 +414,6 @@ static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { tfree(pQueryNode->pExtInfo); tfree(pQueryNode->pSchema); tfree(pQueryNode->info.name); - - tfree(pQueryNode->tableInfo.tableName); // dropAllExprInfo(pQueryNode->pExpr); if (pQueryNode->pPrevNodes != NULL) { @@ -447,9 +440,9 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, switch(pQueryNode->info.type) { case QNODE_TABLESCAN: { - STimeWindow* win = (STimeWindow*)pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64 " cols: ", - pQueryNode->tableInfo.tableName, pQueryNode->tableInfo.uid, win->skey, win->ekey); + SQueryTableInfo* pInfo = (SQueryTableInfo*) pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, + pInfo->tableName, pInfo->uid, pInfo->window.skey, pInfo->window.ekey); assert(len1 > 0); len += len1; From 52f952f97c0c446cc55f1923d4cf46564e64f42e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 15 Nov 2021 10:22:01 +0800 Subject: [PATCH 03/16] [td-10564] Refactor projection parse. --- source/libs/parser/src/astValidate.c | 39 ++++++-------------------- source/libs/parser/src/queryInfoUtil.c | 2 +- 2 files changed, 10 insertions(+), 31 deletions(-) diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 2536f8f060..cb215915b8 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -2652,12 +2652,8 @@ static int32_t validateScalarFunctionParamNum(tSqlExpr* pSqlExpr, int32_t functi return code; } -int32_t doAddProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, SColumnIndex* pColIndex, - const char* aliasName, int32_t colId, SMsgBuf* pMsgBuf) { - STableMeta* pTableMeta = getMetaInfo(pQueryInfo, pColIndex->tableIndex)->pTableMeta; - - SSchema* pSchema = getOneColumnSchema(pTableMeta, pColIndex->columnIndex); - +int32_t doAddProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, SSchema* pSchema, const char* aliasName, + int32_t colId, SMsgBuf* pMsgBuf) { const char* name = (aliasName == NULL)? pSchema->name:aliasName; SSchema s = createSchema(pSchema->type, pSchema->bytes, colId, name); @@ -2708,8 +2704,8 @@ static int32_t doAddProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, SColu } for (int32_t j = 0; j < numOfTotalColumns; ++j) { - pIndex->columnIndex = j; - doAddProjectCol(pQueryInfo, startPos + j, pIndex, NULL, getNewResColId(), pMsgBuf); + SSchema* pSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, j); + doAddProjectCol(pQueryInfo, startPos + j, pSchema, NULL, getNewResColId(), pMsgBuf); } return numOfTotalColumns; @@ -2748,11 +2744,9 @@ static SSchema createConstantColumnSchema(SVariant* pVal, const SToken* exprStr, } static int32_t handleTbnameProjection(SQueryStmtInfo* pQueryInfo, tSqlExprItem* pItem, SColumnIndex* pIndex, int32_t startPos, bool outerQuery, SMsgBuf* pMsgBuf) { - const char* msg3 = "tbname not allowed in outer query"; + const char* msg1 = "tbname not allowed in outer query"; SSchema colSchema = {0}; - char* funcName = NULL; - if (outerQuery) { // todo?? STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); @@ -2769,36 +2763,20 @@ static int32_t handleTbnameProjection(SQueryStmtInfo* pQueryInfo, tSqlExprItem* } if (!existed) { - return buildInvalidOperationMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg1); } colSchema = pSchema[pIndex->columnIndex]; - funcName = "project_col"; } else { colSchema = *getTbnameColumnSchema(); - funcName = "project_tag"; } - SSchema resultSchema = colSchema; - resultSchema.colId = getNewResColId(); - - char rawName[TSDB_COL_NAME_LEN] = {0}; - setTokenAndResColumnName(pItem, resultSchema.name, rawName, sizeof(colSchema.name) - 1); - - STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); - SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, pIndex->type, &colSchema); - - SSourceParam param = {0}; - addIntoSourceParam(¶m, NULL, &c); - - doAddOneExprInfo(pQueryInfo, "project_tag", ¶m, startPos, pTableMetaInfo, &colSchema, 0, rawName, true); - return TSDB_CODE_SUCCESS; + return doAddProjectCol(pQueryInfo, startPos, &colSchema, pItem->aliasName, getNewResColId(), pMsgBuf); } int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery, SMsgBuf* pMsgBuf) { const char* msg1 = "tag for normal table query is not allowed"; const char* msg2 = "invalid column name"; - const char* msg3 = "tbname not allowed in outer query"; if (checkForAliasName(pMsgBuf, pItem->aliasName) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -2861,7 +2839,8 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* return buildInvalidOperationMsg(pMsgBuf, msg1); } - doAddProjectCol(pQueryInfo, startPos, &index, pItem->aliasName, getNewResColId(), pMsgBuf); + SSchema* pSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); + doAddProjectCol(pQueryInfo, startPos, pSchema, pItem->aliasName, getNewResColId(), pMsgBuf); } // add the primary timestamp column even though it is not required by user diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 64156f3843..a0afcf65c8 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -330,7 +330,7 @@ SArray* extractFunctionList(SArray* pExprInfoList) { if (pExprInfo->pExpr->nodeType == TEXPR_FUNCTION_NODE) { taosArrayPush(p, &pExprInfo->pExpr->_function.functionName); } else { - taosArrayPush(p, ""); + taosArrayPush(p, "project"); } } From b7f8ee372132323c7648d6f895ca72c617564fbb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 15 Nov 2021 10:24:58 +0800 Subject: [PATCH 04/16] [td-10564] Refactor. --- source/libs/parser/src/astValidate.c | 18 +++++++++--------- src/client/src/tscSQLParser.c | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index cb215915b8..2a0d02dc91 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -2652,19 +2652,19 @@ static int32_t validateScalarFunctionParamNum(tSqlExpr* pSqlExpr, int32_t functi return code; } -int32_t doAddProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, SSchema* pSchema, const char* aliasName, +int32_t doAddOneProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, SSchema* pSchema, const char* aliasName, int32_t colId, SMsgBuf* pMsgBuf) { const char* name = (aliasName == NULL)? pSchema->name:aliasName; SSchema s = createSchema(pSchema->type, pSchema->bytes, colId, name); tExprNode *pNode = NULL; bool keepTableCols = true; + SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); + SToken colNameToken = {.z = pSchema->name, .n = strlen(pSchema->name)}; tSqlExpr sqlNode = {0}; sqlNode.type = SQL_NODE_TABLE_COLUMN; - - SToken colNameToken = {.z = pSchema->name, .n = strlen(pSchema->name)}; sqlNode.columnName = colNameToken; int32_t ret = sqlExprToExprNode(&pNode, &sqlNode, pQueryInfo, pColumnList, &keepTableCols, pMsgBuf); @@ -2692,7 +2692,7 @@ int32_t doAddProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, SSch return TSDB_CODE_SUCCESS; } -static int32_t doAddProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos, SMsgBuf* pMsgBuf) { +static int32_t doAddMultipleProjectExprAndResColumns(SQueryStmtInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos, SMsgBuf* pMsgBuf) { STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -2705,7 +2705,7 @@ static int32_t doAddProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, SColu for (int32_t j = 0; j < numOfTotalColumns; ++j) { SSchema* pSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, j); - doAddProjectCol(pQueryInfo, startPos + j, pSchema, NULL, getNewResColId(), pMsgBuf); + doAddOneProjectCol(pQueryInfo, startPos + j, pSchema, NULL, getNewResColId(), pMsgBuf); } return numOfTotalColumns; @@ -2771,7 +2771,7 @@ static int32_t handleTbnameProjection(SQueryStmtInfo* pQueryInfo, tSqlExprItem* colSchema = *getTbnameColumnSchema(); } - return doAddProjectCol(pQueryInfo, startPos, &colSchema, pItem->aliasName, getNewResColId(), pMsgBuf); + return doAddOneProjectCol(pQueryInfo, startPos, &colSchema, pItem->aliasName, getNewResColId(), pMsgBuf); } int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery, SMsgBuf* pMsgBuf) { @@ -2796,11 +2796,11 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* if (index.tableIndex == COLUMN_INDEX_INITIAL_VAL) { // all table columns are required. for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { index.tableIndex = i; - int32_t inc = doAddProjectionExprAndResColumn(pQueryInfo, &index, startPos, pMsgBuf); + int32_t inc = doAddMultipleProjectExprAndResColumns(pQueryInfo, &index, startPos, pMsgBuf); startPos += inc; } } else { - doAddProjectionExprAndResColumn(pQueryInfo, &index, startPos, pMsgBuf); + doAddMultipleProjectExprAndResColumns(pQueryInfo, &index, startPos, pMsgBuf); } // add the primary timestamp column even though it is not required by user @@ -2840,7 +2840,7 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* } SSchema* pSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - doAddProjectCol(pQueryInfo, startPos, pSchema, pItem->aliasName, getNewResColId(), pMsgBuf); + doAddOneProjectCol(pQueryInfo, startPos, pSchema, pItem->aliasName, getNewResColId(), pMsgBuf); } // add the primary timestamp column even though it is not required by user diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 05e4cbaa97..4fc78d73e4 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -63,7 +63,7 @@ typedef struct SConvertFunc { int32_t execFuncId; } SConvertFunc; -static SExprInfo* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex, int32_t colId); +static SExprInfo* doAddOneProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex, int32_t colId); static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); @@ -1890,7 +1890,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 } static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSqlExprItem* pItem, int32_t colId) { - SExprInfo* pExpr = doAddProjectCol(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex, colId); + SExprInfo* pExpr = doAddOneProjectCol(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex, colId); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -2157,7 +2157,7 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi return TSDB_CODE_SUCCESS; } -SExprInfo* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex, int32_t colId) { +SExprInfo* doAddOneProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex, int32_t colId) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; int32_t numOfCols = tscGetNumOfColumns(pTableMeta); @@ -2218,7 +2218,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum } for (int32_t j = 0; j < numOfTotalColumns; ++j) { - SExprInfo* pExpr = doAddProjectCol(pQueryInfo, j, pIndex->tableIndex, getNewResColId(pCmd)); + SExprInfo* pExpr = doAddOneProjectCol(pQueryInfo, j, pIndex->tableIndex, getNewResColId(pCmd)); tstrncpy(pExpr->base.aliasName, pSchema[j].name, sizeof(pExpr->base.aliasName)); pIndex->columnIndex = j; From fadd69c652f1039541be364edfbef3862100ac69 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 15 Nov 2021 10:51:22 +0800 Subject: [PATCH 05/16] [td-10564] refactor and add test cases. --- source/libs/parser/src/astValidate.c | 56 ++++++++++++++-------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 2a0d02dc91..5d1ff936dd 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -2652,14 +2652,12 @@ static int32_t validateScalarFunctionParamNum(tSqlExpr* pSqlExpr, int32_t functi return code; } +// todo merge with the addScalarExprAndResColumn int32_t doAddOneProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, SSchema* pSchema, const char* aliasName, int32_t colId, SMsgBuf* pMsgBuf) { const char* name = (aliasName == NULL)? pSchema->name:aliasName; SSchema s = createSchema(pSchema->type, pSchema->bytes, colId, name); - tExprNode *pNode = NULL; - bool keepTableCols = true; - SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); SToken colNameToken = {.z = pSchema->name, .n = strlen(pSchema->name)}; @@ -2667,7 +2665,9 @@ int32_t doAddOneProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, S sqlNode.type = SQL_NODE_TABLE_COLUMN; sqlNode.columnName = colNameToken; - int32_t ret = sqlExprToExprNode(&pNode, &sqlNode, pQueryInfo, pColumnList, &keepTableCols, pMsgBuf); + tExprNode* pNode = NULL; + bool keepTableCols = true; + int32_t ret = sqlExprToExprNode(&pNode, &sqlNode, pQueryInfo, pColumnList, &keepTableCols, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(pNode, NULL); return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select clause"); @@ -2689,6 +2689,12 @@ int32_t doAddOneProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, S } pExpr->base.numOfCols = num; + + if (pQueryInfo->exprListLevelIndex == 0) { + int32_t exists = getNumOfFields(&pQueryInfo->fieldsInfo); + addResColumnInfo(pQueryInfo, exists, &pExpr->base.resSchema, pExpr); + } + return TSDB_CODE_SUCCESS; } @@ -2923,17 +2929,29 @@ static tExprNode* doCreateColumnNode(SQueryStmtInfo* pQueryInfo, SColumnIndex* p pExpr->nodeType = TEXPR_COL_NODE; pExpr->pSchema = calloc(1, sizeof(SSchema)); - SSchema* pSchema = getOneColumnSchema(pTableMeta, pIndex->columnIndex); + + SSchema* pSchema = NULL; + if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + pSchema = getTbnameColumnSchema(); + } else { + pSchema = getOneColumnSchema(pTableMeta, pIndex->columnIndex); + } + *(SSchema*)(pExpr->pSchema) = *pSchema; - if (keepTableCols) { + if (keepTableCols && TSDB_COL_IS_NORMAL_COL(pIndex->type)) { SColumn c = createColumn(pTableMeta->uid, pTableMetaInfo->aliasName, pIndex->type, pExpr->pSchema); taosArrayPush(pCols, &c); } - columnListInsert(pQueryInfo->colList, pTableMeta->uid, pSchema, TSDB_COL_NORMAL); - SSchema* pTsSchema = getOneColumnSchema(pTableMeta, 0); - insertPrimaryTsColumn(pQueryInfo->colList, pTsSchema->name, pTableMeta->uid); + if (TSDB_COL_IS_NORMAL_COL(pIndex->type)) { + columnListInsert(pQueryInfo->colList, pTableMeta->uid, pSchema, TSDB_COL_NORMAL); + SSchema* pTsSchema = getOneColumnSchema(pTableMeta, 0); + insertPrimaryTsColumn(pQueryInfo->colList, pTsSchema->name, pTableMeta->uid); + } else { + columnListInsert(pTableMetaInfo->tagColList, pTableMeta->uid, pSchema, TSDB_COL_TAG); + } + return pExpr; } @@ -3231,26 +3249,6 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStm return TSDB_CODE_SUCCESS; } -static int32_t multiColumnListInsert(SQueryStmtInfo* pQueryInfo, SArray* pColumnList, SMsgBuf* pMsgBuf) { - const char* msg1 = "tag can not be used in expression"; - - SColumn* p1 = taosArrayGet(pColumnList, 0); - - size_t numOfNode = taosArrayGetSize(pColumnList); - for(int32_t k = 0; k < numOfNode; ++k) { - SColumn* p = taosArrayGet(pColumnList, k); - if (TSDB_COL_IS_TAG(p->flag)) { - return buildInvalidOperationMsg(pMsgBuf, msg1); - } - - SSchema s = createSchema(p->info.type, p->info.bytes, p->info.colId, p->name); - columnListInsert(pQueryInfo->colList, p->uid, &s, p->flag); - } - - insertPrimaryTsColumn(pQueryInfo->colList, NULL, p1->uid); - return TSDB_CODE_SUCCESS; -} - static int32_t addScalarExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem, SMsgBuf* pMsgBuf) { SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); SSchema s = createSchema(TSDB_DATA_TYPE_DOUBLE, sizeof(double), getNewResColId(), ""); From bfd23c61d83cd711f2f21662d983ced415350cce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Nov 2021 18:57:00 +0800 Subject: [PATCH 06/16] [td-10564] 1. refactor and fix bug in order by validation. 2. Support the compatible model of ver. 2.0 --- include/common/common.h | 15 +- include/common/taosmsg.h | 2 +- include/common/tglobal.h | 1 + include/libs/function/function.h | 3 +- include/libs/parser/parser.h | 2 +- source/common/src/tglobal.c | 5 + source/libs/function/src/taggfunction.c | 2 +- source/libs/function/src/tfunction.c | 101 ++++------ source/libs/parser/src/astValidate.c | 255 +++++++++++++++++++----- source/libs/parser/src/queryInfoUtil.c | 7 +- source/libs/parser/test/parserTests.cpp | 30 ++- source/libs/parser/test/plannerTest.cpp | 9 +- source/libs/planner/src/planner.c | 156 +++++++-------- 13 files changed, 373 insertions(+), 215 deletions(-) diff --git a/include/common/common.h b/include/common/common.h index d99e4a78b7..1c3a0e22f7 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -55,11 +55,18 @@ typedef struct SDataBlockInfo { int64_t uid; } SDataBlockInfo; +typedef struct SConstantItem { + SColumnInfo info; + int32_t startIndex; // run-length-encoding to save the space for multiple rows + int32_t endIndex; + SVariant value; +} SConstantItem; + typedef struct SSDataBlock { SColumnDataAgg *pBlockAgg; - SArray *pDataBlock; // SArray - SArray *pTagsList; // SArray for tag value - SDataBlockInfo info; + SArray *pDataBlock; // SArray + SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. + SDataBlockInfo info; } SSDataBlock; typedef struct SColumnInfoData { @@ -83,7 +90,7 @@ typedef struct SLimit { typedef struct SOrder { uint32_t order; - int32_t orderColId; + SColumn col; } SOrder; typedef struct SGroupbyExpr { diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 4718d0e4b3..a4bf388309 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -277,7 +277,7 @@ typedef struct SSchema { uint8_t type; char name[TSDB_COL_NAME_LEN]; int16_t colId; - int16_t bytes; + int32_t bytes; } SSchema; //#endif diff --git a/include/common/tglobal.h b/include/common/tglobal.h index f3fce8becd..4296160c8b 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -58,6 +58,7 @@ extern int32_t tsCompressColData; extern int32_t tsMaxNumOfDistinctResults; extern char tsTempDir[]; extern int64_t tsMaxVnodeQueuedBytes; +extern int tsCompatibleModel; // 2.0 compatible model //query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 92d8c972f5..d7360a81bc 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -229,7 +229,7 @@ typedef struct SScalarFunctionInfo { typedef struct SMultiFunctionsDesc { bool stableQuery; bool groupbyColumn; - bool simpleAgg; + bool agg; bool arithmeticOnAgg; bool projectionQuery; bool hasFilter; @@ -261,6 +261,7 @@ int32_t qIsBuiltinFunction(const char* name, int32_t len, bool* scalarFunction); bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId); bool qIsAggregateFunction(const char* functionName); +bool qIsSelectivityFunction(const char* functionName); tExprNode* exprTreeFromBinary(const void* data, size_t size); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 2dec203ce8..0e6c352d71 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -86,7 +86,7 @@ typedef struct SQueryStmtInfo { SLimit slimit; STagCond tagCond; SArray * colCond; - SOrder order; + SArray * order; int16_t numOfTables; int16_t curTableIdx; STableMetaInfo **pTableMetaInfo; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4f30327c06..b4d74dd298 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -79,6 +79,11 @@ int32_t tsCompressMsgSize = -1; */ int32_t tsCompressColData = -1; +/* + * denote if 3.0 query pattern compatible for 2.0 + */ +int32_t tsCompatibleModel = 1; + // client int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index dc6eadf7d8..00f59e8b87 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -4589,7 +4589,7 @@ SAggFunctionInfo aggFunc[35] = {{ }, { // 16 - "ts", + "dummy", FUNCTION_TYPE_AGG, FUNCTION_TS, FUNCTION_TS, diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c index 7e3b4bcd5d..9e70b9a68d 100644 --- a/source/libs/function/src/tfunction.c +++ b/source/libs/function/src/tfunction.c @@ -54,6 +54,18 @@ bool qIsAggregateFunction(const char* functionName) { return !scalarfunc; } +bool qIsSelectivityFunction(const char* functionName) { + assert(functionName != NULL); + pthread_once(&functionHashTableInit, doInitFunctionHashTable); + + size_t len = strlen(functionName); + SAggFunctionInfo** pInfo = taosHashGet(functionHashTable, functionName, len); + if (pInfo != NULL) { + return ((*pInfo)->status | FUNCSTATE_SELECTIVITY) != 0; + } + + return false; +} SAggFunctionInfo* qGetFunctionInfo(const char* name, int32_t len) { pthread_once(&functionHashTableInit, doInitFunctionHashTable); @@ -79,16 +91,17 @@ void qRemoveUdfInfo(uint64_t id, SUdfInfo* pUdfInfo) { bool isTagsQuery(SArray* pFunctionIdList) { int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); for (int32_t i = 0; i < num; ++i) { - int16_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i); + char* f = *(char**) taosArrayGet(pFunctionIdList, i); + + // todo handle count(tbname) query + if (strcmp(f, "project") != 0 && strcmp(f, "count") != 0) { + return false; + } // "select count(tbname)" query // if (functId == FUNCTION_COUNT && pExpr->base.colpDesc->colId == TSDB_TBNAME_COLUMN_INDEX) { // continue; // } - - if (f != FUNCTION_TAGPRJ && f != FUNCTION_TID_TAG) { - return false; - } } return true; @@ -113,23 +126,13 @@ bool isTagsQuery(SArray* pFunctionIdList) { bool isProjectionQuery(SArray* pFunctionIdList) { int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); for (int32_t i = 0; i < num; ++i) { - int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i); - if (f == FUNCTION_TS_DUMMY) { - continue; - } - - if (f != FUNCTION_PRJ && - f != FUNCTION_TAGPRJ && - f != FUNCTION_TAG && - f != FUNCTION_TS && - f != FUNCTION_ARITHM && - f != FUNCTION_DIFF && - f != FUNCTION_DERIVATIVE) { - return false; + char* f = *(char**) taosArrayGet(pFunctionIdList, i); + if (strcmp(f, "project") == 0) { + return true; } } - return true; + return false; } bool isDiffDerivativeQuery(SArray* pFunctionIdList) { @@ -190,11 +193,11 @@ bool isTopBotQuery(SArray* pFunctionIdList) { int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); for (int32_t i = 0; i < num; ++i) { char* f = *(char**) taosArrayGet(pFunctionIdList, i); - if (f == FUNCTION_TS) { + if (strcmp(f, "project") == 0) { continue; } - if (f == FUNCTION_TOP || f == FUNCTION_BOTTOM) { + if (strcmp(f, "top") == 0 || strcmp(f, "bottom") == 0) { return true; } } @@ -273,49 +276,26 @@ bool needReverseScan(SArray* pFunctionIdList) { return false; } -bool isSimpleAggregateRv(SArray* pFunctionIdList) { -// if (pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0) { -// return false; -// } -// -// if (tscIsDiffDerivQuery(pQueryInfo)) { -// return false; -// } -// -// size_t numOfExprs = getNumOfExprs(pQueryInfo); -// for (int32_t i = 0; i < numOfExprs; ++i) { -// SExprInfo* pExpr = getExprInfo(pQueryInfo, i); -// if (pExpr == NULL) { -// continue; -// } -// -// int32_t functionId = pExpr->base.functionId; -// if (functionId < 0) { -// SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); -// if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { -// return true; -// } -// -// continue; -// } -// -// if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY) { -// continue; -// } -// -// if ((!IS_MULTIOUTPUT(aAggs[functionId].status)) || -// (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_TS_COMP)) { -// return true; -// } -// } +bool isAgg(SArray* pFunctionIdList) { + size_t size = taosArrayGetSize(pFunctionIdList); + for (int32_t i = 0; i < size; ++i) { + char* f = *(char**) taosArrayGet(pFunctionIdList, i); + if (strcmp(f, "project") == 0) { + return false; + } + + if (qIsAggregateFunction(f)) { + return true; + } + } return false; } bool isBlockDistQuery(SArray* pFunctionIdList) { int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); - int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, 0); - return (num == 1 && f == FUNCTION_BLKINFO); + char* f = *(char**) taosArrayGet(pFunctionIdList, 0); + return (num == 1 && strcmp(f, "block_dist") == 0); } bool isTwoStageSTableQuery(SArray* pFunctionIdList, int32_t tableIndex) { @@ -426,8 +406,9 @@ void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc) { return; } - pDesc->projectionQuery = isProjectionQuery(pFunctionIdList); - pDesc->onlyTagQuery = isTagsQuery(pFunctionIdList); +// pDesc->projectionQuery = isProjectionQuery(pFunctionIdList); +// pDesc->onlyTagQuery = isTagsQuery(pFunctionIdList); pDesc->interpQuery = isInterpQuery(pFunctionIdList); pDesc->topbotQuery = isTopBotQuery(pFunctionIdList); + pDesc->agg = isAgg(pFunctionIdList); } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 5d1ff936dd..d9fd822b9b 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include #include "astGenerator.h" #include "function.h" @@ -703,6 +702,8 @@ static int32_t parseSlidingClause(SQueryStmtInfo* pQueryInfo, SToken* pSliding, return TSDB_CODE_SUCCESS; } +static void setTsOutputExprInfo(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, int32_t outputIndex, int32_t tableIndex); + // validate the interval info int32_t validateIntervalNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf* pMsgBuf) { const char* msg1 = "sliding cannot be used without interval"; @@ -720,11 +721,6 @@ int32_t validateIntervalNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMs } } - // orderby column not set yet, set it to be the primary timestamp column - if (pQueryInfo->order.orderColId == INT32_MIN) { - pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_ID; - } - // interval is not null SToken *t = &pSqlNode->interval.interval; if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.interval, @@ -753,6 +749,13 @@ int32_t validateIntervalNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMs return TSDB_CODE_TSC_INVALID_OPERATION; } + if (tsCompatibleModel) { + SExprInfo* pFirstExpr = getExprInfo(pQueryInfo, 0); + if (pFirstExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE || strcasecmp(pFirstExpr->pExpr->_function.functionName, "dummy") != 0) { + setTsOutputExprInfo(pQueryInfo, pTableMetaInfo, 0, 0); + } + } + // It is a time window query pQueryInfo->info.timewindow = true; return TSDB_CODE_SUCCESS; @@ -922,8 +925,6 @@ int32_t validateLimitNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBu } } -static void setTsOutputExprInfo(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, int32_t outputIndex, int32_t tableIndex); - int32_t validateOrderbyNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid column name in orderby clause"; const char* msg2 = "too many order by columns"; @@ -934,6 +935,8 @@ int32_t validateOrderbyNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsg return TSDB_CODE_SUCCESS; } + pQueryInfo->order = taosArrayInit(4, sizeof(SOrder)); + STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, 0); SArray* pSortOrder = pSqlNode->pSortOrder; @@ -944,40 +947,43 @@ int32_t validateOrderbyNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsg * for super table query, the order option must be less than 3. */ size_t size = taosArrayGetSize(pSortOrder); - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo)) { + if ((UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) && (pQueryInfo->info.projectionQuery)) { if (size > 1) { return buildInvalidOperationMsg(pMsgBuf, msg3); } - } else { - if (size > 2) { - return buildInvalidOperationMsg(pMsgBuf, msg2); - } } // handle the first part of order by - SVariant* pVar = taosArrayGet(pSortOrder, 0); - SSchema s = {0}; - if (pVar->nType == TSDB_DATA_TYPE_BINARY) { - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - SToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; - if (getColumnIndexByName(&columnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); - } + for(int32_t i = 0; i < taosArrayGetSize(pSortOrder); ++i) { + SVariant* pVar = taosArrayGet(pSortOrder, i); + if (pVar->nType == TSDB_DATA_TYPE_BINARY) { + SColumn c = {0}; - s = *(SSchema*) getOneColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - } else { // order by [1|2|3] - if (pVar->i > getNumOfFields(&pQueryInfo->fieldsInfo)) { - return buildInvalidOperationMsg(pMsgBuf, msg4); - } + // find the orde column among the result field. + for (int32_t j = 0; j < getNumOfFields(&pQueryInfo->fieldsInfo); ++j) { + SInternalField* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.internalField, j); + SSchema* pSchema = &pInfo->pExpr->base.resSchema; + if (strcasecmp(pVar->pz, pSchema->name) == 0) { + setColumn(&c, pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, TSDB_COL_TMP, &pSchema); + return TSDB_CODE_SUCCESS; + } + } - SExprInfo* pExprInfo = getExprInfo(pQueryInfo, pVar->i); - s = pExprInfo->base.resSchema; + return buildInvalidOperationMsg(pMsgBuf, "invalid order by column"); + } else { // order by [1|2|3] + if (pVar->i > getNumOfFields(&pQueryInfo->fieldsInfo)) { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } + + int32_t index = pVar->i - 1; + SExprInfo* pExprInfo = getExprInfo(pQueryInfo, index); + + SOrder c = {0}; + setColumn(&c.col, pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, TSDB_COL_TMP, &pExprInfo->base.resSchema); + taosArrayPush(pQueryInfo->order, &c); + } } - SListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); - pQueryInfo->order.order = pItem->sortOrder; - pQueryInfo->order.orderColId = s.colId; - return TSDB_CODE_SUCCESS; } @@ -1378,6 +1384,8 @@ int32_t validateFillNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf return TSDB_CODE_SUCCESS; } +static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo); + int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { assert(pSqlNode != NULL && (pSqlNode->from == NULL || taosArrayGetSize(pSqlNode->from->list) > 0)); @@ -1529,11 +1537,6 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* return TSDB_CODE_TSC_INVALID_OPERATION; } - // set order by info - if (validateOrderbyNode(pQueryInfo, pSqlNode, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - // set interval value if (validateIntervalNode(pQueryInfo, pSqlNode, pMsgBuf) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -1558,6 +1561,11 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* return TSDB_CODE_SUCCESS; } + // set order by info + if (validateOrderbyNode(pQueryInfo, pSqlNode, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + if ((code = validateLimitNode(pQueryInfo, pSqlNode, pMsgBuf)) != TSDB_CODE_SUCCESS) { return code; } @@ -1567,7 +1575,9 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* } } - for(int32_t i = 0; i < getExprFunctionLevel(pQueryInfo); ++i) { + exprInfoPushDown(pQueryInfo); + + for(int32_t i = 0; i < 1; ++i) { SArray* functionList = extractFunctionList(pQueryInfo->exprList[i]); extractFunctionDesc(functionList, &pQueryInfo->info); @@ -1579,6 +1589,87 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* return TSDB_CODE_SUCCESS; // Does not build query message here } +static bool isTagOrPrimaryTs(SExprInfo* pExprInfo) { + if (pExprInfo->pExpr->nodeType != TEXPR_COL_NODE) { + return false; + } + + assert(pExprInfo->base.pColumns->info.colId == pExprInfo->pExpr->pSchema->colId); + return (TSDB_COL_IS_TAG(pExprInfo->base.pColumns->flag) || pExprInfo->pExpr->pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID); +} + +// todo extract the table column in expression + +static bool isGroupbyCol(SExprInfo* pExprInfo, SGroupbyExpr* pGroupbyExpr) { + assert(pExprInfo != NULL && pGroupbyExpr != NULL); + + int32_t nodeType = pExprInfo->pExpr->nodeType; + assert(nodeType == TEXPR_COL_NODE || nodeType == TEXPR_BINARYEXPR_NODE); + + for(int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) { + SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i); + if (pCol->info.colId == pExprInfo->pExpr->pSchema->colId) { + return true; + } + } + + return false; +} + +static bool isAllAggExpr(SArray* pList) { + assert(pList != NULL); + + for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) { + SExprInfo* p = taosArrayGetP(pList, k); + if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) { + return false; + } + } + + return true; +} + +static SExprInfo* doCreateColumnNodeFromAggFunc(SSchema* pSchema); + +static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo) { + assert(pQueryInfo != NULL); + + size_t level = getExprFunctionLevel(pQueryInfo); + for(int32_t i = 0; i < level - 1; ++i) { + SArray* p = pQueryInfo->exprList[i]; + + SArray* pNext = pQueryInfo->exprList[i + 1]; + if (!isAllAggExpr(pNext)) { + continue; + } + + for (int32_t j = 0; j < taosArrayGetSize(p); ++j) { + SExprInfo* pExpr = taosArrayGetP(p, j); + + if (pExpr->pExpr->nodeType == TEXPR_FUNCTION_NODE && qIsAggregateFunction(pExpr->pExpr->_function.functionName)) { + bool canPushDown = true; + for (int32_t k = 0; k < taosArrayGetSize(pNext); ++k) { + SExprInfo* pNextLevelExpr = taosArrayGetP(pNext, k); + if (pExpr->base.pColumns->info.colId == pNextLevelExpr->base.resSchema.colId) { + // pExpr is dependent on the output of the under layer, so it can not be push downwards + canPushDown = false; + break; + } + } + + if (canPushDown) { + taosArrayInsert(pNext, j, &pExpr); + taosArrayRemove(p, j); + + // todo add the project function in level of "i" + SExprInfo* pNew = doCreateColumnNodeFromAggFunc(&pExpr->base.resSchema); + taosArrayInsert(p, j, &pNew); + } + } + } + } +} + int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { assert(pQueryInfo != NULL && pMsgBuf != NULL); @@ -1597,9 +1688,18 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { // 1. invalid sql: // select top(col, k) from table_name [interval(1d)|session(ts, 1d)|statewindow(col)] order by k asc // order by normal column is not supported - int32_t colId = pQueryInfo->order.orderColId; - if (pQueryInfo->info.timewindow && colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - return buildInvalidOperationMsg(pMsgBuf, msg2); + if (pQueryInfo->order != NULL) { + size_t numOfOrder = taosArrayGetSize(pQueryInfo->order); + if (numOfOrder > 1) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + if (numOfOrder > 0) { + SColumn* pOrderCol = taosArrayGet(pQueryInfo->order, 0); + if (pQueryInfo->info.timewindow && pOrderCol->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + } } // select top(col, k) from table_name interval(10s) fill(prev) @@ -1609,14 +1709,25 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { } // select top(col, k), count(*) from table_name - int32_t num = 0; - SExprInfo* pMain = NULL; size_t size = getNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { SExprInfo* pExpr = getExprInfo(pQueryInfo, i); - const char* functionName = pExpr->pExpr->_function.functionName; - if (strcmp(functionName, "top") != 0 && strcmp(functionName, "bottom") != 0) { + if (pExpr->pExpr->nodeType == TEXPR_COL_NODE) { + if (!isTagOrPrimaryTs(pExpr) && !isGroupbyCol(pExpr, &pQueryInfo->groupbyExpr)) { + return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select clause"); + } + + } else if (pExpr->pExpr->nodeType == TEXPR_BINARYEXPR_NODE) { + continue; + // todo extract all column node in tree, and check for each node + + continue; + } + + // dummy column is also the placeholder for primary timestamp column in the result. + const char* functionName = pExpr->pExpr->_function.functionName; + if (strcmp(functionName, "top") != 0 && strcmp(functionName, "bottom") != 0 && strcmp(functionName, "dummy") != 0) { if (qIsAggregateFunction(functionName)) { return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select clause"); } @@ -1709,6 +1820,35 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { pQueryInfo->info.groupbyColumn) { return buildInvalidOperationMsg(pMsgBuf, msg9); } + + /* + * 9. invalid sql: + * select count(*), col_name from table_name + */ + if (pQueryInfo->info.agg) { + bool isSelectivity = false; + + if (pQueryInfo->info.projectionQuery) { + size_t size = getNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { + SExprInfo* pExpr = getExprInfo(pQueryInfo, i); + if (pExpr->pExpr->nodeType == TEXPR_FUNCTION_NODE) { + if (!isSelectivity) { + isSelectivity = qIsSelectivityFunction(pExpr->pExpr->_function.functionName); + } + continue; + } + + if (isSelectivity && isTagOrPrimaryTs(pExpr)) { + continue; + } + + if (!isGroupbyCol(pExpr, &pQueryInfo->groupbyExpr)) { + return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select"); + } + } + } + } } int32_t addResColumnInfo(SQueryStmtInfo* pQueryInfo, int32_t outputIndex, SSchema* pSchema, SExprInfo* pSqlExpr) { @@ -1856,7 +1996,7 @@ static void setTsOutputExprInfo(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTab SSourceParam param = {0}; addIntoSourceParam(¶m, NULL, &col); - SExprInfo* pExpr = createExprInfo(pTableMetaInfo, "ts_dummy", ¶m, &s, TSDB_KEYSIZE); + SExprInfo* pExpr = createExprInfo(pTableMetaInfo, "dummy", ¶m, &s, TSDB_KEYSIZE); strncpy(pExpr->base.token, "ts", tListLen(pExpr->base.token)); SArray* pExprList = getCurrentExprList(pQueryInfo); @@ -2695,6 +2835,7 @@ int32_t doAddOneProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, S addResColumnInfo(pQueryInfo, exists, &pExpr->base.resSchema, pExpr); } + pQueryInfo->info.projectionQuery = true; return TSDB_CODE_SUCCESS; } @@ -2926,9 +3067,9 @@ static tExprNode* doCreateColumnNode(SQueryStmtInfo* pQueryInfo, SColumnIndex* p STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; tExprNode* pExpr = calloc(1, sizeof(tExprNode)); - pExpr->nodeType = TEXPR_COL_NODE; - pExpr->pSchema = calloc(1, sizeof(SSchema)); + pExpr->nodeType = TEXPR_COL_NODE; + pExpr->pSchema = calloc(1, sizeof(SSchema)); SSchema* pSchema = NULL; if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { @@ -2955,6 +3096,23 @@ static tExprNode* doCreateColumnNode(SQueryStmtInfo* pQueryInfo, SColumnIndex* p return pExpr; } +static SExprInfo* doCreateColumnNodeFromAggFunc(SSchema* pSchema) { + tExprNode* pExprNode = calloc(1, sizeof(tExprNode)); + + pExprNode->nodeType = TEXPR_COL_NODE; + pExprNode->pSchema = calloc(1, sizeof(SSchema)); + *(SSchema*)(pExprNode->pSchema) = *pSchema; + + SExprInfo* pExpr = calloc(1, sizeof(SExprInfo)); + if (pExpr == NULL) { + return NULL; + } + + pExpr->pExpr = pExprNode; + memcpy(&pExpr->base.resSchema, pSchema, sizeof(SSchema)); + return pExpr; +} + static int32_t validateSqlExpr(const tSqlExpr* pSqlExpr, SQueryStmtInfo *pQueryInfo, SMsgBuf* pMsgBuf); static int32_t doProcessFunctionLeafNodeParam(SQueryStmtInfo* pQueryInfo, int32_t* num, tExprNode*** p, SArray* pCols, @@ -3366,7 +3524,6 @@ int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) { // use the dynamic array list to decide if the function is valid or not // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 - // todo refacto to remove this function if ((code = addProjectionExprAndResColumn(pQueryInfo, pItem, outerQuery, pMsgBuf)) != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index a0afcf65c8..c548f1556a 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -325,12 +325,15 @@ SArray* extractFunctionList(SArray* pExprInfoList) { size_t len = taosArrayGetSize(pExprInfoList); SArray* p = taosArrayInit(len, POINTER_BYTES); + for(int32_t i = 0; i < len; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pExprInfoList, i); if (pExprInfo->pExpr->nodeType == TEXPR_FUNCTION_NODE) { - taosArrayPush(p, &pExprInfo->pExpr->_function.functionName); + char* name = strdup(pExprInfo->pExpr->_function.functionName); + taosArrayPush(p, &name); } else { - taosArrayPush(p, "project"); + char* name = strdup("project"); + taosArrayPush(p, &name); } } diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 5a240061b2..2193a44604 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -16,6 +16,7 @@ #include #include #include +#include "tglobal.h" #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" @@ -422,8 +423,8 @@ TEST(testCase, function_Test10) { sqlCheck("select block_dist() + 20 from `t.1abc`", true); sqlCheck("select count(b), c from `t.1abc`", false); sqlCheck("select top(a, 20), count(b) from `t.1abc`", false); -// sqlCheck("select top(a, 20), b from `t.1abc`", false); -// sqlCheck("select top(a, 20), a+20 from `t.1abc`", true); + sqlCheck("select top(a, 20), b from `t.1abc`", false); + sqlCheck("select top(a, 20), a+20 from `t.1abc`", true); // sqlCheck("select top(a, 20), bottom(a, 10) from `t.1abc`", false); // sqlCheck("select last_row(*), count(b) from `t.1abc`", false); // sqlCheck("select last_row(a, b) + 20 from `t.1abc`", false); @@ -457,9 +458,14 @@ TEST(testCase, function_Test6) { ASSERT_EQ(ret, 0); SArray* pExprList = pQueryInfo->exprList[0]; - ASSERT_EQ(taosArrayGetSize(pExprList), 5); + if (tsCompatibleModel) { + ASSERT_EQ(taosArrayGetSize(pExprList), 6); + } else { + ASSERT_EQ(taosArrayGetSize(pExprList), 5); + } - SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); + int32_t index = tsCompatibleModel? 1:0; + SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, index); ASSERT_EQ(p1->base.pColumns->uid, 110); ASSERT_EQ(p1->base.numOfParams, 0); ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); @@ -477,9 +483,12 @@ TEST(testCase, function_Test6) { ASSERT_STREQ(pParam->pSchema->name, "t.1abc.a+b"); ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 5); - SExprInfo* p2 = (SExprInfo*)taosArrayGetP(pExprList, 1); + int32_t numOfResCol = tsCompatibleModel? 6:5; + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, numOfResCol); + + index = tsCompatibleModel? 2:1; + SExprInfo* p2 = (SExprInfo*)taosArrayGetP(pExprList, index); ASSERT_EQ(p2->base.pColumns->uid, 110); ASSERT_EQ(p2->base.numOfParams, 0); ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); @@ -527,9 +536,10 @@ TEST(testCase, function_Test6) { ASSERT_EQ(ret, 0); SArray* pExprList = pQueryInfo->exprList[0]; - ASSERT_EQ(taosArrayGetSize(pExprList), 2); + ASSERT_EQ(taosArrayGetSize(pExprList), 3); - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); + int32_t index = tsCompatibleModel? 1:0; + SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, index); ASSERT_EQ(p1->base.pColumns->uid, 110); ASSERT_EQ(p1->base.numOfParams, 0); ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); @@ -553,7 +563,9 @@ TEST(testCase, function_Test6) { ASSERT_EQ(pParam->pSchema->colId, p2->base.resSchema.colId); ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); + + int32_t numOfCols = tsCompatibleModel? 3:2; + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, numOfCols); destroyQueryInfo(pQueryInfo); qParserClearupMetaRequestInfo(&req); diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index ee2c01dc48..cf8463f245 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -63,7 +64,6 @@ void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { setSchema(&pSchema[1], TSDB_DATA_TYPE_INT, 4, "a", 1); setSchema(&pSchema[2], TSDB_DATA_TYPE_DOUBLE, 8, "b", 2); setSchema(&pSchema[3], TSDB_DATA_TYPE_DOUBLE, 8, "col", 3); - } void generateLogicplan(const char* sql) { @@ -132,7 +132,9 @@ TEST(testCase, planner_test) { ASSERT_EQ(ret, 0); SArray* pExprList = pQueryInfo->exprList[0]; - ASSERT_EQ(taosArrayGetSize(pExprList), 2); + + int32_t num = tsCompatibleModel? 2:1; + ASSERT_EQ(taosArrayGetSize(pExprList), num); SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); ASSERT_EQ(p1->base.pColumns->uid, 110); @@ -172,6 +174,7 @@ TEST(testCase, displayPlan) { generateLogicplan("select count(A+B) from `t.1abc` group by a"); generateLogicplan("select count(length(a)+b) from `t.1abc` group by a"); generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s)"); + generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s) order by 1 desc "); generateLogicplan("select count(*),sum(a),avg(b),min(a+b)+99 from `t.1abc`"); generateLogicplan("select count(*), min(a) + 99 from `t.1abc`"); generateLogicplan("select count(length(count(*) + 22)) from `t.1abc`"); @@ -180,7 +183,7 @@ TEST(testCase, displayPlan) { generateLogicplan("select count(*), first(a), last(b) from `t.1abc` session(ts, 20s)"); // order by + group by column + limit offset + fill - + generateLogicplan("select top(a, 20) k from `t.1abc` order by k asc limit 3 offset 1"); // join diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 101ea3ec2f..8e53f231ef 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -46,14 +46,12 @@ typedef struct SJoinCond { static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); -static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo); int32_t qOptimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { return 0; } int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { - exprInfoPushDown((struct SQueryStmtInfo*) pQueryInfo); SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); assert(taosArrayGetSize(upstream) == 1); @@ -164,6 +162,12 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit)); break; } + + case QNODE_SORT: { + pNode->pExtInfo = taosArrayDup(pExtInfo); + break; + } + default: break; } @@ -267,6 +271,11 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQuer pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, NULL, 0, pInfo); } + if (pQueryInfo->order != NULL) { + SArray* pList = pQueryInfo->exprList[0]; + pNode = createQueryNode(QNODE_SORT, "Sort", &pNode, 1, pList->pData, taosArrayGetSize(pList), pQueryInfo->order); + } + if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) { pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit); } @@ -306,44 +315,6 @@ static bool isAllAggExpr(SArray* pList) { return true; } -static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo) { - assert(pQueryInfo != NULL); - - size_t level = getExprFunctionLevel(pQueryInfo); - for(int32_t i = 0; i < level - 1; ++i) { - SArray* p = pQueryInfo->exprList[i]; - - SArray* pNext = pQueryInfo->exprList[i + 1]; - if (!isAllAggExpr(pNext)) { - continue; - } - - for (int32_t j = 0; j < taosArrayGetSize(p); ++j) { - SExprInfo* pExpr = taosArrayGetP(p, j); - - if (pExpr->pExpr->nodeType == TEXPR_FUNCTION_NODE && qIsAggregateFunction(pExpr->pExpr->_function.functionName)) { - bool canPushDown = true; - for (int32_t k = 0; k < taosArrayGetSize(pNext); ++k) { - SExprInfo* pNextLevelExpr = taosArrayGetP(pNext, k); - if (pExpr->base.pColumns->info.colId == pNextLevelExpr->base.resSchema.colId) { - // pExpr is dependent on the output of the under layer, so it can not be push downwards - canPushDown = false; - break; - } - } - - if (canPushDown) { - taosArrayInsert(pNext, j, &pExpr); - taosArrayRemove(p, j); - - // todo add the project function in level of "i" - - } - } - } - } -} - SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { SArray* upstream = NULL; @@ -440,13 +411,13 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, switch(pQueryNode->info.type) { case QNODE_TABLESCAN: { - SQueryTableInfo* pInfo = (SQueryTableInfo*) pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, - pInfo->tableName, pInfo->uid, pInfo->window.skey, pInfo->window.ekey); + SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, pInfo->tableName, pInfo->uid, + pInfo->window.skey, pInfo->window.ekey); assert(len1 > 0); len += len1; - for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i); len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId); @@ -467,7 +438,7 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, len += len1; - for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* p = &pExprInfo->base; @@ -485,18 +456,18 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, len1 = sprintf(buf + len, ")"); len += len1; - //todo print filter info + // todo print filter info len1 = sprintf(buf + len, " filters:(nil)\n"); len += len1; break; } case QNODE_AGGREGATE: { - for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* pExpr = &pExprInfo->base; - len += sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); + len += sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len, ", "); len += len1; @@ -509,43 +480,43 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_TIMEWINDOW: { - for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* pExpr = &pExprInfo->base; - len += sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); + len += sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len,", "); + len1 = sprintf(buf + len, ", "); len += len1; } } - len1 = sprintf(buf + len,") "); + len1 = sprintf(buf + len, ") "); len += len1; SInterval* pInterval = pQueryNode->pExtInfo; // todo dynamic return the time precision len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "(%s)\n", - pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, TSDB_TIME_PRECISION_MILLI_STR, - pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR); + pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, + TSDB_TIME_PRECISION_MILLI_STR, pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR); len += len1; break; } case QNODE_STATEWINDOW: { - for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - SSqlExpr* pExpr = &pExprInfo->base; - len += sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); + SSqlExpr* pExpr = &pExprInfo->base; + len += sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len,", "); + len1 = sprintf(buf + len, ", "); len += len1; } } - len1 = sprintf(buf + len,") "); + len1 = sprintf(buf + len, ") "); len += len1; SColumn* pCol = pQueryNode->pExtInfo; @@ -555,44 +526,44 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_SESSIONWINDOW: { - for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - SSqlExpr* pExpr = &pExprInfo->base; - len += sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); + SSqlExpr* pExpr = &pExprInfo->base; + len += sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len,", "); + len1 = sprintf(buf + len, ", "); len += len1; } } - len1 = sprintf(buf + len,") "); + len1 = sprintf(buf + len, ") "); len += len1; struct SSessionWindow* ps = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "col:[%s #%d], gap:%"PRId64" (ms) \n", ps->col.name, ps->col.info.colId, ps->gap); + len1 = sprintf(buf + len, "col:[%s #%d], gap:%" PRId64 " (ms) \n", ps->col.name, ps->col.info.colId, ps->gap); len += len1; break; } - case QNODE_GROUPBY: { // todo hide the invisible column - for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + case QNODE_GROUPBY: { + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* pExpr = &pExprInfo->base; - len1 = sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); + len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); len += len1; if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len,", "); + len1 = sprintf(buf + len, ", "); len += len1; } } SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; - len1 = sprintf(buf + len,") groupby_col: "); + len1 = sprintf(buf + len, ") groupby_col: "); len += len1; - for(int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) { SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i); len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId); len += len1; @@ -604,61 +575,78 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, case QNODE_FILL: { SFillEssInfo* pEssInfo = pQueryNode->pExtInfo; - len1 = sprintf(buf + len,"%d", pEssInfo->fillType); + len1 = sprintf(buf + len, "%d", pEssInfo->fillType); len += len1; if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) { - len1 = sprintf(buf + len,", val:"); + len1 = sprintf(buf + len, ", val:"); len += len1; // todo get the correct fill data type - for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - len1 = sprintf(buf + len,"%"PRId64, pEssInfo->val[i]); + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + len1 = sprintf(buf + len, "%" PRId64, pEssInfo->val[i]); len += len1; if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len,", "); + len1 = sprintf(buf + len, ", "); len += len1; } } } - len1 = sprintf(buf + len,")\n"); + len1 = sprintf(buf + len, ")\n"); len += len1; break; } case QNODE_LIMIT: { SLimit* pVal = pQueryNode->pExtInfo; - len1 = sprintf(buf + len,"limit: %"PRId64", offset: %"PRId64")\n", pVal->limit, pVal->offset); + len1 = sprintf(buf + len, "limit: %" PRId64 ", offset: %" PRId64 ")\n", pVal->limit, pVal->offset); len += len1; break; } case QNODE_DISTINCT: case QNODE_TAGSCAN: { - len1 = sprintf(buf + len,"cols: "); + len1 = sprintf(buf + len, "cols: "); len += len1; - for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - SSchema* resSchema = &pExprInfo->base.resSchema; + SSchema* resSchema = &pExprInfo->base.resSchema; - len1 = sprintf(buf + len,"[%s #%d]", resSchema->name, resSchema->colId); + len1 = sprintf(buf + len, "[%s #%d]", resSchema->name, resSchema->colId); len += len1; if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len,", "); + len1 = sprintf(buf + len, ", "); len += len1; } } - len1 = sprintf(buf + len,")\n"); + len1 = sprintf(buf + len, ")\n"); len += len1; break; } + case QNODE_SORT: { + len1 = sprintf(buf + len, "cols:"); + len += len1; + + SArray* pSort = pQueryNode->pExtInfo; + for (int32_t i = 0; i < taosArrayGetSize(pSort); ++i) { + SOrder* p = taosArrayGet(pSort, i); + len1 = sprintf(buf + len, " [%s #%d %s]", p->col.name, p->col.info.colId, p->order == TSDB_ORDER_ASC? "ASC":"DESC"); + + len += len1; + } + + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + case QNODE_JOIN: { // print join condition len1 = sprintf(buf + len, ")\n"); From 28cd6afcef3a25d8f3fbddbde924f0225ebab42e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Nov 2021 10:18:08 +0800 Subject: [PATCH 07/16] [td-10564] Fix bug in parse orderby clause. --- source/libs/function/src/texpr.c | 10 ++ source/libs/parser/src/astGenerator.c | 3 +- source/libs/parser/src/astValidate.c | 109 +++++++++++++++++---- source/libs/parser/test/plannerTest.cpp | 13 ++- source/libs/planner/src/planner.c | 121 +++++++----------------- 5 files changed, 144 insertions(+), 112 deletions(-) diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index 6970b85638..70a7e9973f 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "function.h" #include "os.h" #include "exception.h" @@ -550,6 +551,15 @@ tExprNode* exprdup(tExprNode* pNode) { } else if (pNode->nodeType == TEXPR_COL_NODE) { pCloned->pSchema = calloc(1, sizeof(SSchema)); *pCloned->pSchema = *pNode->pSchema; + } else if (pNode->nodeType == TEXPR_FUNCTION_NODE) { + strcpy(pCloned->_function.functionName, pNode->_function.functionName); + + int32_t num = pNode->_function.num; + pCloned->_function.num = num; + pCloned->_function.pChild = calloc(num, POINTER_BYTES); + for(int32_t i = 0; i < num; ++i) { + pCloned->_function.pChild[i] = exprdup(pNode->_function.pChild[i]); + } } pCloned->nodeType = pNode->nodeType; diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index 53d05c87b3..f328486556 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -59,8 +59,7 @@ SArray *tListItemAppendToken(SArray *pList, SToken *pAliasToken, uint8_t sortOrd if (pAliasToken) { SListItem item; - assert(0); -// taosVariantCreate(&item.pVar, pAliasToken); + taosVariantCreate(&item.pVar, pAliasToken->z, pAliasToken->n, pAliasToken->type); item.sortOrder = sortOrder; taosArrayPush(pList, &item); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index d9fd822b9b..7b6c423fb6 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -954,22 +954,32 @@ int32_t validateOrderbyNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsg } // handle the first part of order by + bool found = false; for(int32_t i = 0; i < taosArrayGetSize(pSortOrder); ++i) { - SVariant* pVar = taosArrayGet(pSortOrder, i); + SListItem* pItem = taosArrayGet(pSortOrder, i); + + SVariant* pVar = &pItem->pVar; if (pVar->nType == TSDB_DATA_TYPE_BINARY) { - SColumn c = {0}; + SOrder order = {0}; // find the orde column among the result field. for (int32_t j = 0; j < getNumOfFields(&pQueryInfo->fieldsInfo); ++j) { SInternalField* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.internalField, j); SSchema* pSchema = &pInfo->pExpr->base.resSchema; if (strcasecmp(pVar->pz, pSchema->name) == 0) { - setColumn(&c, pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, TSDB_COL_TMP, &pSchema); - return TSDB_CODE_SUCCESS; + setColumn(&order.col, pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, TSDB_COL_TMP, pSchema); + + order.order = pItem->sortOrder; + taosArrayPush(pQueryInfo->order, &order); + found = true; + break; } } - return buildInvalidOperationMsg(pMsgBuf, "invalid order by column"); + if (!found) { + return buildInvalidOperationMsg(pMsgBuf, "invalid order by column"); + } + } else { // order by [1|2|3] if (pVar->i > getNumOfFields(&pQueryInfo->fieldsInfo)) { return buildInvalidOperationMsg(pMsgBuf, msg4); @@ -980,6 +990,7 @@ int32_t validateOrderbyNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsg SOrder c = {0}; setColumn(&c.col, pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, TSDB_COL_TMP, &pExprInfo->base.resSchema); + c.order = pItem->sortOrder; taosArrayPush(pQueryInfo->order, &c); } } @@ -1248,16 +1259,17 @@ int32_t checkForInvalidOrderby(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, S #endif static int32_t checkFillQueryRange(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { - const char* msg3 = "start(end) time of time range required or time range too large"; + const char* msg1 = "start(end) time of time range required or time range too large"; if (pQueryInfo->interval.interval == 0) { return TSDB_CODE_SUCCESS; } - bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); - if (initialWindows) { - return buildInvalidOperationMsg(pMsgBuf, msg3); - } + // TODO disable this check temporarily +// bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); +// if (initialWindows) { +// return buildInvalidOperationMsg(pMsgBuf, msg1); +// } int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); @@ -1267,7 +1279,7 @@ static int32_t checkFillQueryRange(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) // number of result is not greater than 10,000,000 if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { - return buildInvalidOperationMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg1); } } @@ -1384,7 +1396,8 @@ int32_t validateFillNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf return TSDB_CODE_SUCCESS; } -static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo); +static void pushDownAggFuncExprInfo(SQueryStmtInfo* pQueryInfo); +static void addColumnNodeFromLowerLevel(SQueryStmtInfo* pQueryInfo); int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { assert(pSqlNode != NULL && (pSqlNode->from == NULL || taosArrayGetSize(pSqlNode->from->list) > 0)); @@ -1575,7 +1588,8 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* } } - exprInfoPushDown(pQueryInfo); + pushDownAggFuncExprInfo(pQueryInfo); +// addColumnNodeFromLowerLevel(pQueryInfo); for(int32_t i = 0; i < 1; ++i) { SArray* functionList = extractFunctionList(pQueryInfo->exprList[i]); @@ -1629,15 +1643,29 @@ static bool isAllAggExpr(SArray* pList) { return true; } -static SExprInfo* doCreateColumnNodeFromAggFunc(SSchema* pSchema); +static bool isAllProjectExpr(SArray *pList) { + assert(pList != NULL); -static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo) { + for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { + SExprInfo* p = taosArrayGetP(pList, i); + if (p->pExpr->nodeType == TEXPR_FUNCTION_NODE && !qIsAggregateFunction(p->pExpr->_function.functionName)) { + return false; + } + } + + return true; +} + +static SExprInfo* createColumnNodeFromAggFunc(SSchema* pSchema); + +static void pushDownAggFuncExprInfo(SQueryStmtInfo* pQueryInfo) { assert(pQueryInfo != NULL); size_t level = getExprFunctionLevel(pQueryInfo); for(int32_t i = 0; i < level - 1; ++i) { SArray* p = pQueryInfo->exprList[i]; + // If direct lower level expressions are all aggregate function, check if current function can be push down or not SArray* pNext = pQueryInfo->exprList[i + 1]; if (!isAllAggExpr(pNext)) { continue; @@ -1650,8 +1678,8 @@ static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo) { bool canPushDown = true; for (int32_t k = 0; k < taosArrayGetSize(pNext); ++k) { SExprInfo* pNextLevelExpr = taosArrayGetP(pNext, k); + // pExpr depends on the output of the down level, so it can not be push downwards if (pExpr->base.pColumns->info.colId == pNextLevelExpr->base.resSchema.colId) { - // pExpr is dependent on the output of the under layer, so it can not be push downwards canPushDown = false; break; } @@ -1661,8 +1689,8 @@ static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo) { taosArrayInsert(pNext, j, &pExpr); taosArrayRemove(p, j); - // todo add the project function in level of "i" - SExprInfo* pNew = doCreateColumnNodeFromAggFunc(&pExpr->base.resSchema); + // Add the project function of the current level, to output the calculated result + SExprInfo* pNew = createColumnNodeFromAggFunc(&pExpr->base.resSchema); taosArrayInsert(p, j, &pNew); } } @@ -1670,6 +1698,49 @@ static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo) { } } +// todo change the logic plan data +static void addColumnNodeFromLowerLevel(SQueryStmtInfo* pQueryInfo) { + assert(pQueryInfo != NULL); + + size_t level = getExprFunctionLevel(pQueryInfo); + for (int32_t i = 0; i < level - 1; ++i) { + SArray* p = pQueryInfo->exprList[i]; + if (isAllAggExpr(p)) { + continue; + } + + // If direct lower level expressions are all aggregate function, check if current function can be push down or not + SArray* pNext = pQueryInfo->exprList[i + 1]; + if (isAllAggExpr(pNext)) { + continue; + } + + for (int32_t j = 0; j < taosArrayGetSize(pNext); ++j) { + SExprInfo* pExpr = taosArrayGetP(p, j); + + bool exists = false; + for (int32_t k = 0; k < taosArrayGetSize(p); ++k) { + SExprInfo* pNextLevelExpr = taosArrayGetP(pNext, k); + // pExpr depends on the output of the down level, so it can not be push downwards + if (pExpr->base.pColumns->info.colId == pNextLevelExpr->base.resSchema.colId) { + exists = true; + break; + } + } + + if (!exists) { + SExprInfo* pNew = calloc(1, sizeof(SExprInfo)); + pNew->pExpr = exprdup(pExpr->pExpr); + memcpy(&pNew->base, &pExpr->base, sizeof(SSqlExpr)); + + int32_t pos = taosArrayGetSize(p); + // Add the project function of the current level, to output the calculated result + taosArrayInsert(p, pos - 1, &pExpr); + } + } + } +} + int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { assert(pQueryInfo != NULL && pMsgBuf != NULL); @@ -3096,7 +3167,7 @@ static tExprNode* doCreateColumnNode(SQueryStmtInfo* pQueryInfo, SColumnIndex* p return pExpr; } -static SExprInfo* doCreateColumnNodeFromAggFunc(SSchema* pSchema) { +static SExprInfo* createColumnNodeFromAggFunc(SSchema* pSchema) { tExprNode* pExprNode = calloc(1, sizeof(tExprNode)); pExprNode->nodeType = TEXPR_COL_NODE; diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index cf8463f245..c86e687664 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -182,15 +182,18 @@ TEST(testCase, displayPlan) { generateLogicplan("select count(*), first(a), last(b) from `t.1abc` state_window(a)"); generateLogicplan("select count(*), first(a), last(b) from `t.1abc` session(ts, 20s)"); - // order by + group by column + limit offset + fill + // order by + group by column + limit offset generateLogicplan("select top(a, 20) k from `t.1abc` order by k asc limit 3 offset 1"); + // fill + generateLogicplan("select min(a) from `t.1abc` where ts>now and ts= 0; --i) { SArray* p = pQueryInfo->exprList[i]; + size_t num = taosArrayGetSize(p); - size_t num = taosArrayGetSize(p); bool aggregateFunc = false; for(int32_t j = 0; j < num; ++j) { SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0); @@ -265,10 +266,11 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQuer if (pQueryInfo->fillType != TSDB_FILL_NONE) { SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); pInfo->fillType = pQueryInfo->fillType; - pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t)); + pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t)); memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); - pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, NULL, 0, pInfo); + SArray* p = pQueryInfo->exprList[0]; // top expression in select clause + pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo); } if (pQueryInfo->order != NULL) { @@ -433,26 +435,11 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_PROJECT: { - len1 = sprintf(buf + len, "cols: "); + len1 = sprintf(buf + len, "cols:"); assert(len1 > 0); - len += len1; - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - - SSqlExpr* p = &pExprInfo->base; - len1 = sprintf(buf + len, "[%s #%d]", p->resSchema.name, p->resSchema.colId); - assert(len1 > 0); - - len += len1; - - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - + len = printExprInfo(buf, pQueryNode, len); len1 = sprintf(buf + len, ")"); len += len1; @@ -463,34 +450,15 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_AGGREGATE: { - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - - SSqlExpr* pExpr = &pExprInfo->base; - len += sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - + len = printExprInfo(buf, pQueryNode, len); len1 = sprintf(buf + len, ")\n"); len += len1; + break; } case QNODE_TIMEWINDOW: { - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - - SSqlExpr* pExpr = &pExprInfo->base; - len += sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - + len = printExprInfo(buf, pQueryNode, len); len1 = sprintf(buf + len, ") "); len += len1; @@ -506,16 +474,7 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_STATEWINDOW: { - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - SSqlExpr* pExpr = &pExprInfo->base; - len += sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - + len = printExprInfo(buf, pQueryNode, len); len1 = sprintf(buf + len, ") "); len += len1; @@ -526,15 +485,7 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_SESSIONWINDOW: { - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - SSqlExpr* pExpr = &pExprInfo->base; - len += sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } + len = printExprInfo(buf, pQueryNode, len); len1 = sprintf(buf + len, ") "); len += len1; @@ -546,18 +497,7 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_GROUPBY: { - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - - SSqlExpr* pExpr = &pExprInfo->base; - len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - - len += len1; - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } + len = printExprInfo(buf, pQueryNode, len); SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; len1 = sprintf(buf + len, ") groupby_col: "); @@ -611,18 +551,7 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, len1 = sprintf(buf + len, "cols: "); len += len1; - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - SSchema* resSchema = &pExprInfo->base.resSchema; - - len1 = sprintf(buf + len, "[%s #%d]", resSchema->name, resSchema->colId); - len += len1; - - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } + len = printExprInfo(buf, pQueryNode, len); len1 = sprintf(buf + len, ")\n"); len += len1; @@ -658,6 +587,26 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, return len; } +int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len) { + int32_t len1 = 0; + + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); + + SSqlExpr* pExpr = &pExprInfo->base; + len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); + assert(len1 > 0); + + len += len1; + if (i < pQueryNode->numOfExpr - 1) { + len1 = sprintf(buf + len, ", "); + len += len1; + } + } + + return len; +} + int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen); From ff86bc296ee924d23eb293fdb788080dafbe1b8b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 5 Dec 2021 20:28:27 +0800 Subject: [PATCH 08/16] TD-10431 rename test file --- source/dnode/mgmt/impl/test/CMakeLists.txt | 2 +- .../test/{test01 => profile}/CMakeLists.txt | 14 +-- .../test01.cpp => profile/profile.cpp} | 85 ++++++++++++------- source/dnode/mgmt/impl/test/sut/deploy.cpp | 16 ++-- source/dnode/mgmt/impl/test/sut/deploy.h | 2 +- 5 files changed, 69 insertions(+), 50 deletions(-) rename source/dnode/mgmt/impl/test/{test01 => profile}/CMakeLists.txt (61%) rename source/dnode/mgmt/impl/test/{test01/test01.cpp => profile/profile.cpp} (50%) diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 95ffa548b2..79ae5631c2 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1 +1 @@ -add_subdirectory(test01) \ No newline at end of file +add_subdirectory(profile) \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/test01/CMakeLists.txt b/source/dnode/mgmt/impl/test/profile/CMakeLists.txt similarity index 61% rename from source/dnode/mgmt/impl/test/test01/CMakeLists.txt rename to source/dnode/mgmt/impl/test/profile/CMakeLists.txt index 5f88e9c116..2e99a2e53d 100644 --- a/source/dnode/mgmt/impl/test/test01/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/profile/CMakeLists.txt @@ -1,20 +1,20 @@ -add_executable(dndTest01 "") +add_executable(dndTestProfile "") -target_sources(dndTest01 +target_sources(dndTestProfile PRIVATE - "test01.cpp" + "profile.cpp" "../sut/deploy.cpp" ) target_link_libraries( - dndTest01 + dndTestProfile PUBLIC dnode PUBLIC util PUBLIC os PUBLIC gtest_main ) -target_include_directories(dndTest01 +target_include_directories(dndTestProfile PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" @@ -24,6 +24,6 @@ target_include_directories(dndTest01 enable_testing() add_test( - NAME dndTest01 - COMMAND dndTest01 + NAME dndTestProfile + COMMAND dndTestProfile ) diff --git a/source/dnode/mgmt/impl/test/test01/test01.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp similarity index 50% rename from source/dnode/mgmt/impl/test/test01/test01.cpp rename to source/dnode/mgmt/impl/test/profile/profile.cpp index 6c440074dc..9bbb648e66 100644 --- a/source/dnode/mgmt/impl/test/test01/test01.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -15,22 +15,32 @@ #include "deploy.h" -class DndTest01 : public ::testing::Test { +class DndTestProfile : public ::testing::Test { protected: - void SetUp() override { - pServer = createServer("/tmp/dndTest01"); + void SetUp() override {} + void TearDown() override {} + + static void SetUpTestSuite() { + pServer = createServer("/tmp/dndTestProfile", "localhost", 9527); pClient = createClient("root", "taosdata"); } - void TearDown() override { + + static void TearDownTestSuite() { dropServer(pServer); dropClient(pClient); } - SServer* pServer; - SClient* pClient; + static SServer* pServer; + static SClient* pClient; }; -TEST_F(DndTest01, connectMsg) { +SServer* DndTestProfile::pServer; +SClient* DndTestProfile::pClient; + +TEST_F(DndTestProfile, connectMsg_01) { + ASSERT_NE(pServer, nullptr); + ASSERT_NE(pClient, nullptr); + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); pReq->pid = htonl(1234); strcpy(pReq->app, "test01"); @@ -63,33 +73,42 @@ TEST_F(DndTest01, connectMsg) { EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } -// TEST_F(DndTest01, heartbeatMsg) { -// SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); -// pReq->connId = htonl(1); -// pReq->pid = htonl(1234); -// pReq->numOfQueries = htonl(0); -// pReq->numOfStreams = htonl(0); -// strcpy(pReq->app, "test01"); +TEST_F(DndTestProfile, heartbeatMsg_01) { + ASSERT_NE(pServer, nullptr); + ASSERT_NE(pClient, nullptr); -// SRpcMsg rpcMsg = {0}; -// rpcMsg.pCont = pReq; -// rpcMsg.contLen = sizeof(SHeartBeatMsg); -// rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; + SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); + pReq->connId = htonl(1); + pReq->pid = htonl(1234); + pReq->numOfQueries = htonl(0); + pReq->numOfStreams = htonl(0); + strcpy(pReq->app, "test01"); -// sendMsg(pClient, &rpcMsg); + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SHeartBeatMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; -// SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pClient->pRsp; -// ASSERT(pRsp); - // pRsp->epSet.port[0] = htonl(pRsp->epSet.port[0]); + sendMsg(pClient, &rpcMsg); -// EXPECT_EQ(htonl(pRsp->connId), 1); -// EXPECT_GT(htonl(pRsp->queryId), 0); -// EXPECT_GT(htonl(pRsp->streamId), 1); -// EXPECT_EQ(htonl(pRsp->totalDnodes), 1); -// EXPECT_EQ(htonl(pRsp->onlineDnodes), 1); -// EXPECT_EQ(pRsp->killConnection, 0); - // EXPECT_EQ(pRsp->epSet.inUse, 0); - // EXPECT_EQ(pRsp->epSet.numOfEps, 1); - // EXPECT_EQ(pRsp->epSet.port[0], 9527); - // EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); -// } + SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->connId = htonl(pRsp->connId); + pRsp->queryId = htonl(pRsp->queryId); + pRsp->streamId = htonl(pRsp->streamId); + pRsp->totalDnodes = htonl(pRsp->totalDnodes); + pRsp->onlineDnodes = htonl(pRsp->onlineDnodes); + pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); + + EXPECT_EQ(pRsp->connId, 1); + EXPECT_EQ(pRsp->queryId, 0); + EXPECT_EQ(pRsp->streamId, 0); + EXPECT_EQ(pRsp->totalDnodes, 1); + EXPECT_EQ(pRsp->onlineDnodes, 1); + EXPECT_EQ(pRsp->killConnection, 0); + + EXPECT_EQ(pRsp->epSet.inUse, 0); + EXPECT_EQ(pRsp->epSet.numOfEps, 1); + EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); +} diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 132ce54ec6..239120aa88 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -15,7 +15,7 @@ #include "deploy.h" -void initLog(char *path) { +void initLog(char* path) { mDebugFlag = 207; char temp[PATH_MAX]; snprintf(temp, PATH_MAX, "%s/taosdlog", path); @@ -32,7 +32,7 @@ void* runServer(void* param) { } } -void initOption(SDnodeOpt* pOption, char *path) { +void initOption(SDnodeOpt* pOption, char* path, char* fqdn, uint16_t port) { pOption->sver = 1; pOption->numOfCores = 1; pOption->numOfSupportMnodes = 1; @@ -44,19 +44,19 @@ void initOption(SDnodeOpt* pOption, char *path) { pOption->ratioOfQueryCores = 1; pOption->maxShellConns = 1000; pOption->shellActivityTimer = 30; - pOption->serverPort = 9527; + pOption->serverPort = port; strcpy(pOption->dataDir, path); - strcpy(pOption->localEp, "localhost:9527"); - strcpy(pOption->localFqdn, "localhost"); - strcpy(pOption->firstEp, "localhost:9527"); + snprintf(pOption->localEp, TSDB_EP_LEN, "%s:&u", fqdn, port); + snprintf(pOption->localFqdn, TSDB_FQDN_LEN, "%s", fqdn); + snprintf(pOption->firstEp, TSDB_EP_LEN, "%s:&u", fqdn, port); taosRemoveDir(path); taosMkDir(path); } -SServer* createServer(char *path) { +SServer* createServer(char* path, char* fqdn, uint16_t port) { SDnodeOpt option = {0}; - initOption(&option, path); + initOption(&option, path, fqdn, port); SDnode* pDnode = dndInit(&option); ASSERT(pDnode); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.h b/source/dnode/mgmt/impl/test/sut/deploy.h index 6c47b03fe7..b830c7351e 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.h +++ b/source/dnode/mgmt/impl/test/sut/deploy.h @@ -36,7 +36,7 @@ typedef struct { tsem_t sem; } SClient; -SServer* createServer(char* path); +SServer* createServer(char* path, char *fqdn, uint16_t port); void dropServer(SServer* pServer); SClient* createClient(char *user, char *pass); void dropClient(SClient* pClient); From 57a722a6d8e3b0493e0b6c1cf8771f487a747186 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 5 Dec 2021 23:28:11 +0800 Subject: [PATCH 09/16] TD-10431 rename test file --- include/common/taosmsg.h | 14 +++- .../dnode/mgmt/impl/test/profile/profile.cpp | 5 +- source/dnode/mnode/impl/src/mndProfile.c | 71 ++++++++----------- 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index e14788be47..93186252a7 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -893,8 +893,18 @@ typedef struct { } SHeartBeatRsp; typedef struct { - char queryId[TSDB_KILL_MSG_LEN + 1]; -} SKillQueryMsg, SKillConnMsg; + int32_t connId; + int32_t streamId; +} SKillStreamMsg; + +typedef struct { + int32_t connId; + int32_t queryId; +} SKillQueryMsg; + +typedef struct { + int32_t connId; +} SKillConnMsg; typedef struct { char user[TSDB_USER_LEN]; diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 9bbb648e66..ebdaa85ac7 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -21,7 +21,8 @@ class DndTestProfile : public ::testing::Test { void TearDown() override {} static void SetUpTestSuite() { - pServer = createServer("/tmp/dndTestProfile", "localhost", 9527); + pServer = createServer("/tmp/dndTestProfile", "localhost", 7100); + ASSERT(pServer); pClient = createClient("root", "taosdata"); } @@ -38,7 +39,6 @@ SServer* DndTestProfile::pServer; SClient* DndTestProfile::pClient; TEST_F(DndTestProfile, connectMsg_01) { - ASSERT_NE(pServer, nullptr); ASSERT_NE(pClient, nullptr); SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); @@ -74,7 +74,6 @@ TEST_F(DndTestProfile, connectMsg_01) { } TEST_F(DndTestProfile, heartbeatMsg_01) { - ASSERT_NE(pServer, nullptr); ASSERT_NE(pClient, nullptr); SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index bdd0ab500a..ac606b2ff6 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -84,6 +84,12 @@ int32_t mndInitProfile(SMnode *pMnode) { mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndGetConnsMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndGetQueryMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndCancelGetNextQuery); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndGetStreamMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStreams); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream); return 0; } @@ -122,15 +128,19 @@ static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t int32_t keepTime = pMnode->shellActivityTimer * 3; SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); - - mDebug("conn:%d, is created, user:%s", connId, user); - return pConn; + if (pConn == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("conn:%d, failed to put into cache since %s, user:%s", connId, user, terrstr()); + return NULL; + } else { + mDebug("conn:%d, is created, user:%s", connId, user); + return pConn; + } } static void mndFreeConn(SConnObj *pConn) { tfree(pConn->pQueries); tfree(pConn->pStreams); - tfree(pConn); mDebug("conn:%d, is destroyed", pConn->connId); } @@ -224,6 +234,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { SConnectRsp *pRsp = rpcMallocCont(sizeof(SConnectRsp)); if (pRsp == NULL) { + mndReleaseConn(pMnode, pConn); terrno = TSDB_CODE_OUT_OF_MEMORY; mError("user:%s, failed to login from %s while create rsp since %s", pMsg->user, ip, terrstr()); return -1; @@ -241,6 +252,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { pRsp->clusterId = htonl(pMnode->clusterId); pRsp->connId = htonl(pConn->connId); mndGetMnodeEpSet(pMnode, &pRsp->epSet); + mndReleaseConn(pMnode, pConn); pMsg->contLen = sizeof(SConnectRsp); pMsg->pCont = pRsp; @@ -354,28 +366,17 @@ static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { mndReleaseUser(pMnode, pUser); SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; + int32_t connId = htonl(pKill->connId); + int32_t queryId = htonl(pKill->queryId); mInfo("kill query msg is received, queryId:%s", pKill->queryId); - const char delim = ':'; - char *connIdStr = strtok(pKill->queryId, &delim); - char *queryIdStr = strtok(NULL, &delim); - - if (queryIdStr == NULL || connIdStr == NULL) { - mError("failed to kill query, queryId:%s", pKill->queryId); - terrno = TSDB_CODE_MND_INVALID_QUERY_ID; - return -1; - } - - int32_t queryId = (int32_t)strtol(queryIdStr, NULL, 10); - - int32_t connId = atoi(connIdStr); SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); if (pConn == NULL) { - mError("connId:%s, failed to kill queryId:%d, conn not exist", connIdStr, queryId); + mError("connId:%d, failed to kill queryId:%d, conn not exist", connId, queryId); terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%s, queryId:%d is killed by user:%s", connIdStr, queryId, pMsg->user); + mInfo("connId:%d, queryId:%d is killed by user:%s", connId, queryId, pMsg->user); pConn->queryId = queryId; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); return 0; @@ -395,29 +396,18 @@ static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg) { } mndReleaseUser(pMnode, pUser); - SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; - mInfo("kill stream msg is received, streamId:%s", pKill->queryId); - - const char delim = ':'; - char *connIdStr = strtok(pKill->queryId, &delim); - char *streamIdStr = strtok(NULL, &delim); - - if (streamIdStr == NULL || connIdStr == NULL) { - mError("failed to kill stream, streamId:%s", pKill->queryId); - terrno = TSDB_CODE_MND_INVALID_STREAM_ID; - return -1; - } - - int32_t streamId = (int32_t)strtol(streamIdStr, NULL, 10); - int32_t connId = atoi(connIdStr); + SKillStreamMsg *pKill = pMsg->rpcMsg.pCont; + int32_t connId = htonl(pKill->connId); + int32_t streamId = htonl(pKill->streamId); + mDebug("kill stream msg is received, streamId:%s", streamId); SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); if (pConn == NULL) { - mError("connId:%s, failed to kill streamId:%d, conn not exist", connIdStr, streamId); + mError("connId:%d, failed to kill streamId:%d, conn not exist", connId, streamId); terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%s, streamId:%d is killed by user:%s", connIdStr, streamId, pMsg->user); + mInfo("connId:%d, streamId:%d is killed by user:%s", connId, streamId, pMsg->user); pConn->streamId = streamId; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); return TSDB_CODE_SUCCESS; @@ -438,14 +428,15 @@ static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { mndReleaseUser(pMnode, pUser); SKillConnMsg *pKill = pMsg->rpcMsg.pCont; - int32_t connId = atoi(pKill->queryId); - SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); + int32_t connId = htonl(pKill->connId); + + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); if (pConn == NULL) { - mError("connId:%s, failed to kill, conn not exist", pKill->queryId); + mError("connId:%s, failed to kill connection, conn not exist", connId); terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%s, is killed by user:%s", pKill->queryId, pMsg->user); + mInfo("connId:%s, is killed by user:%s", connId, pMsg->user); pConn->killed = 1; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); return TSDB_CODE_SUCCESS; From c0fc32f6c00df18d2e83d11ed6236d996b89397a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 6 Dec 2021 14:38:37 +0800 Subject: [PATCH 10/16] TD-10431 profile test --- include/common/taosmsg.h | 120 ++--- .../dnode/mgmt/impl/test/profile/profile.cpp | 428 +++++++++++++++++- source/dnode/mgmt/impl/test/sut/deploy.cpp | 48 +- source/dnode/mgmt/impl/test/sut/deploy.h | 6 +- source/dnode/mnode/impl/inc/mndDef.h | 1 - source/dnode/mnode/impl/src/mndProfile.c | 17 +- source/dnode/mnode/impl/src/mndShow.c | 165 ++++--- 7 files changed, 613 insertions(+), 172 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 93186252a7..9443ef7475 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -549,56 +549,61 @@ typedef struct { // todo: the show handle should be replaced with id typedef struct { SMsgHead header; - union{uint64_t qhandle; uint64_t qId;}; // query handle - uint16_t free; + union { + int32_t showId; + int64_t qhandle; + int64_t qId; + }; // query handle + int8_t free; } SRetrieveTableMsg; typedef struct SRetrieveTableRsp { int32_t numOfRows; - int8_t completed; // all results are returned to client - int16_t precision; - int64_t offset; // updated offset value for multi-vnode projection query + int64_t offset; // updated offset value for multi-vnode projection query int64_t useconds; + int8_t completed; // all results are returned to client + int8_t precision; int8_t compressed; + int8_t reserved; int32_t compLen; char data[]; } SRetrieveTableRsp; typedef struct { - char db[TSDB_FULL_DB_NAME_LEN]; - int32_t cacheBlockSize; //MB - int32_t totalBlocks; - int32_t maxTables; - int32_t daysPerFile; - int32_t daysToKeep0; - int32_t daysToKeep1; - int32_t daysToKeep2; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; - int32_t commitTime; - int32_t fsyncPeriod; - uint8_t precision; // time resolution - int8_t compression; - int8_t walLevel; - int8_t replications; - int8_t quorum; - int8_t ignoreExist; - int8_t update; - int8_t cacheLastRow; - int8_t dbType; - int16_t partitions; - int8_t reserve[5]; + char db[TSDB_FULL_DB_NAME_LEN]; + int32_t cacheBlockSize; // MB + int32_t totalBlocks; + int32_t maxTables; + int32_t daysPerFile; + int32_t daysToKeep0; + int32_t daysToKeep1; + int32_t daysToKeep2; + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; + int32_t commitTime; + int32_t fsyncPeriod; + int8_t precision; // time resolution + int8_t compression; + int8_t walLevel; + int8_t replications; + int8_t quorum; + int8_t ignoreExist; + int8_t update; + int8_t cacheLastRow; + int8_t dbType; + int16_t partitions; + int8_t reserve[5]; } SCreateDbMsg, SAlterDbMsg; typedef struct { - char name[TSDB_FUNC_NAME_LEN]; - char path[PATH_MAX]; - int32_t funcType; - uint8_t outputType; - int16_t outputLen; - int32_t bufSize; - int32_t codeLen; - char code[]; + char name[TSDB_FUNC_NAME_LEN]; + char path[PATH_MAX]; + int32_t funcType; + int8_t outputType; + int16_t outputLen; + int32_t bufSize; + int32_t codeLen; + char code[]; } SCreateFuncMsg; typedef struct { @@ -626,8 +631,8 @@ typedef struct { } SDropFuncMsg; typedef struct { - char db[TSDB_TABLE_FNAME_LEN]; - uint8_t ignoreNotExists; + char db[TSDB_TABLE_FNAME_LEN]; + int8_t ignoreNotExists; } SDropDbMsg, SUseDbMsg, SSyncDbMsg; typedef struct { @@ -762,21 +767,20 @@ typedef struct { } SVgroupsMsg, SVgroupsInfo; typedef struct STableMetaMsg { - int32_t contLen; - char tableFname[TSDB_TABLE_FNAME_LEN]; // table id - uint8_t numOfTags; - uint8_t precision; - uint8_t tableType; - int16_t numOfColumns; - int16_t sversion; - int16_t tversion; - int32_t tid; - uint64_t uid; - SVgroupMsg vgroup; - - char sTableName[TSDB_TABLE_FNAME_LEN]; - uint64_t suid; - SSchema schema[]; + int32_t contLen; + char tableFname[TSDB_TABLE_FNAME_LEN]; // table id + int8_t numOfTags; + int8_t precision; + int8_t tableType; + int16_t numOfColumns; + int16_t sversion; + int16_t tversion; + int32_t tid; + int64_t uid; + SVgroupMsg vgroup; + char sTableName[TSDB_TABLE_FNAME_LEN]; + int64_t suid; + SSchema schema[]; } STableMetaMsg; typedef struct SMultiTableMeta { @@ -802,10 +806,10 @@ typedef struct { * payloadLen is the length of payload */ typedef struct { - int8_t type; - char db[TSDB_FULL_DB_NAME_LEN]; - uint16_t payloadLen; - char payload[]; + int8_t type; + char db[TSDB_FULL_DB_NAME_LEN]; + int16_t payloadLen; + char payload[]; } SShowMsg; typedef struct { @@ -815,7 +819,7 @@ typedef struct { } SCompactMsg; typedef struct SShowRsp { - uint64_t qhandle; + int32_t showId; STableMetaMsg tableMeta; } SShowRsp; diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index ebdaa85ac7..210885bd27 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -21,9 +21,15 @@ class DndTestProfile : public ::testing::Test { void TearDown() override {} static void SetUpTestSuite() { - pServer = createServer("/tmp/dndTestProfile", "localhost", 7100); + const char* user = "root"; + const char* pass = "taosdata"; + const char* path = "/tmp/dndTestProfile"; + const char* fqdn = "localhost"; + uint16_t port = 9527; + + pServer = createServer(path, fqdn, port); ASSERT(pServer); - pClient = createClient("root", "taosdata"); + pClient = createClient(user, pass, fqdn, port); } static void TearDownTestSuite() { @@ -33,12 +39,14 @@ class DndTestProfile : public ::testing::Test { static SServer* pServer; static SClient* pClient; + static int32_t connId; }; SServer* DndTestProfile::pServer; SClient* DndTestProfile::pClient; +int32_t DndTestProfile::connId; -TEST_F(DndTestProfile, connectMsg_01) { +TEST_F(DndTestProfile, SConnectMsg_01) { ASSERT_NE(pClient, nullptr); SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); @@ -52,8 +60,10 @@ TEST_F(DndTestProfile, connectMsg_01) { rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); - SConnectRsp* pRsp = (SConnectRsp*)pClient->pRsp->pCont; + SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); pRsp->acctId = htonl(pRsp->acctId); pRsp->clusterId = htonl(pRsp->clusterId); @@ -71,13 +81,165 @@ TEST_F(DndTestProfile, connectMsg_01) { EXPECT_EQ(pRsp->epSet.numOfEps, 1); EXPECT_EQ(pRsp->epSet.port[0], 9527); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); + + connId = pRsp->connId; } -TEST_F(DndTestProfile, heartbeatMsg_01) { +TEST_F(DndTestProfile, SConnectMsg_02) { + ASSERT_NE(pClient, nullptr); + + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + pReq->pid = htonl(1234); + strcpy(pReq->app, "test01"); + strcpy(pReq->db, "invalid_db"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SConnectMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_DB); + ASSERT_EQ(pMsg->contLen, 0); +} + +TEST_F(DndTestProfile, SConnectMsg_03) { + ASSERT_NE(pClient, nullptr); + int32_t showId = 0; + + { + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pReq->type = TSDB_MGMT_TABLE_CONNS; + strcpy(pReq->db, ""); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SShowMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + + SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->showId = htonl(pRsp->showId); + STableMetaMsg* pMeta = &pRsp->tableMeta; + pMeta->contLen = htonl(pMeta->contLen); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tid = htonl(pMeta->tid); + pMeta->uid = htobe64(pMeta->uid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pRsp->showId; + + EXPECT_NE(pRsp->showId, 0); + EXPECT_EQ(pMeta->contLen, 0); + EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->numOfColumns, 7); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tid, 0); + EXPECT_EQ(pMeta->uid, 0); + EXPECT_STREQ(pMeta->sTableName, ""); + EXPECT_EQ(pMeta->suid, 0); + + SSchema* pSchema = NULL; + pSchema = &pMeta->schema[0]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); + EXPECT_EQ(pSchema->bytes, 4); + EXPECT_STREQ(pSchema->name, "connId"); + + pSchema = &pMeta->schema[1]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "user"); + + pSchema = &pMeta->schema[2]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "program"); + + pSchema = &pMeta->schema[3]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); + EXPECT_EQ(pSchema->bytes, 4); + EXPECT_STREQ(pSchema->name, "pid"); + + pSchema = &pMeta->schema[4]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "ip:port"); + + pSchema = &pMeta->schema[5]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); + EXPECT_EQ(pSchema->bytes, 8); + EXPECT_STREQ(pSchema->name, "login_time"); + + pSchema = &pMeta->schema[6]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); + EXPECT_EQ(pSchema->bytes, 8); + EXPECT_STREQ(pSchema->name, "last_access"); + } + + { + SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pReq->showId = htonl(showId); + pReq->free = 0; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SRetrieveTableMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_NE(pMsg->code, 0); + + SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->offset = htobe64(pRsp->offset); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->compLen = htonl(pRsp->compLen); + + EXPECT_EQ(pRsp->numOfRows, 1); + EXPECT_EQ(pRsp->offset, 0); + EXPECT_EQ(pRsp->useconds, 0); + EXPECT_EQ(pRsp->completed, 1); + EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRsp->compressed, 0); + EXPECT_EQ(pRsp->reserved, 0); + EXPECT_EQ(pRsp->compLen, 0); + } +} + +TEST_F(DndTestProfile, SHeartBeatMsg_01) { ASSERT_NE(pClient, nullptr); SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); - pReq->connId = htonl(1); + pReq->connId = htonl(connId); pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); @@ -89,8 +251,10 @@ TEST_F(DndTestProfile, heartbeatMsg_01) { rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); - SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pClient->pRsp->pCont; + SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); pRsp->connId = htonl(pRsp->connId); pRsp->queryId = htonl(pRsp->queryId); @@ -99,7 +263,7 @@ TEST_F(DndTestProfile, heartbeatMsg_01) { pRsp->onlineDnodes = htonl(pRsp->onlineDnodes); pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); - EXPECT_EQ(pRsp->connId, 1); + EXPECT_EQ(pRsp->connId, connId); EXPECT_EQ(pRsp->queryId, 0); EXPECT_EQ(pRsp->streamId, 0); EXPECT_EQ(pRsp->totalDnodes, 1); @@ -111,3 +275,251 @@ TEST_F(DndTestProfile, heartbeatMsg_01) { EXPECT_EQ(pRsp->epSet.port[0], 9527); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } + +TEST_F(DndTestProfile, SKillConnMsg_01) { + ASSERT_NE(pClient, nullptr); + + { + SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(sizeof(SKillConnMsg)); + pReq->connId = htonl(connId); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SKillConnMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_KILL_CONN; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + { + SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); + pReq->connId = htonl(connId); + pReq->pid = htonl(1234); + pReq->numOfQueries = htonl(0); + pReq->numOfStreams = htonl(0); + strcpy(pReq->app, "test01"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SHeartBeatMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_TSC_INVALID_CONNECTION); + ASSERT_EQ(pMsg->contLen, 0); + } + + { + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + pReq->pid = htonl(1234); + strcpy(pReq->app, "test01"); + strcpy(pReq->db, ""); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SConnectMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + + SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->acctId = htonl(pRsp->acctId); + pRsp->clusterId = htonl(pRsp->clusterId); + pRsp->connId = htonl(pRsp->connId); + pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); + + EXPECT_EQ(pRsp->acctId, 1); + EXPECT_GT(pRsp->clusterId, 0); + EXPECT_EQ(pRsp->superAuth, 1); + EXPECT_EQ(pRsp->readAuth, 1); + EXPECT_EQ(pRsp->writeAuth, 1); + + EXPECT_EQ(pRsp->epSet.inUse, 0); + EXPECT_EQ(pRsp->epSet.numOfEps, 1); + EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); + + connId = pRsp->connId; + } +} + +TEST_F(DndTestProfile, SKillConnMsg_02) { + ASSERT_NE(pClient, nullptr); + + SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(sizeof(SKillConnMsg)); + pReq->connId = htonl(2345); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SKillConnMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_KILL_CONN; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); +} + +TEST_F(DndTestProfile, SKillQueryMsg_01) { + ASSERT_NE(pClient, nullptr); + + { + SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(sizeof(SKillQueryMsg)); + pReq->connId = htonl(connId); + pReq->queryId = htonl(1234); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SKillQueryMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_KILL_QUERY; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + ASSERT_EQ(pMsg->contLen, 0); + } + + { + SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); + pReq->connId = htonl(connId); + pReq->pid = htonl(1234); + pReq->numOfQueries = htonl(0); + pReq->numOfStreams = htonl(0); + strcpy(pReq->app, "test01"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SHeartBeatMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + + SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->connId = htonl(pRsp->connId); + pRsp->queryId = htonl(pRsp->queryId); + pRsp->streamId = htonl(pRsp->streamId); + pRsp->totalDnodes = htonl(pRsp->totalDnodes); + pRsp->onlineDnodes = htonl(pRsp->onlineDnodes); + pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); + + EXPECT_EQ(pRsp->connId, connId); + EXPECT_EQ(pRsp->queryId, 1234); + EXPECT_EQ(pRsp->streamId, 0); + EXPECT_EQ(pRsp->totalDnodes, 1); + EXPECT_EQ(pRsp->onlineDnodes, 1); + EXPECT_EQ(pRsp->killConnection, 0); + + EXPECT_EQ(pRsp->epSet.inUse, 0); + EXPECT_EQ(pRsp->epSet.numOfEps, 1); + EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); + } +} + +TEST_F(DndTestProfile, SKillQueryMsg_02) { + ASSERT_NE(pClient, nullptr); + + SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(sizeof(SKillQueryMsg)); + pReq->connId = htonl(2345); + pReq->queryId = htonl(1234); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SKillQueryMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_KILL_QUERY; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); +} + +TEST_F(DndTestProfile, SKillStreamMsg_01) { + ASSERT_NE(pClient, nullptr); + + { + SKillStreamMsg* pReq = (SKillStreamMsg*)rpcMallocCont(sizeof(SKillStreamMsg)); + pReq->connId = htonl(connId); + pReq->streamId = htonl(3579); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SKillStreamMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_KILL_STREAM; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + ASSERT_EQ(pMsg->contLen, 0); + } + + { + SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); + pReq->connId = htonl(connId); + pReq->pid = htonl(1234); + pReq->numOfQueries = htonl(0); + pReq->numOfStreams = htonl(0); + strcpy(pReq->app, "test01"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SHeartBeatMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + + SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->connId = htonl(pRsp->connId); + pRsp->queryId = htonl(pRsp->queryId); + pRsp->streamId = htonl(pRsp->streamId); + pRsp->totalDnodes = htonl(pRsp->totalDnodes); + pRsp->onlineDnodes = htonl(pRsp->onlineDnodes); + pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); + + EXPECT_EQ(pRsp->connId, connId); + EXPECT_EQ(pRsp->queryId, 0); + EXPECT_EQ(pRsp->streamId, 3579); + EXPECT_EQ(pRsp->totalDnodes, 1); + EXPECT_EQ(pRsp->onlineDnodes, 1); + EXPECT_EQ(pRsp->killConnection, 0); + + EXPECT_EQ(pRsp->epSet.inUse, 0); + EXPECT_EQ(pRsp->epSet.numOfEps, 1); + EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); + } +} + +TEST_F(DndTestProfile, SKillStreamMsg_02) { + ASSERT_NE(pClient, nullptr); + + SKillStreamMsg* pReq = (SKillStreamMsg*)rpcMallocCont(sizeof(SKillStreamMsg)); + pReq->connId = htonl(2345); + pReq->streamId = htonl(1234); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SKillStreamMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_KILL_QUERY; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); +} diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 239120aa88..6a7732b7ed 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -15,8 +15,27 @@ #include "deploy.h" -void initLog(char* path) { +void initLog(const char* path) { + dDebugFlag = 0; + vDebugFlag = 0; mDebugFlag = 207; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 0; + sdbDebugFlag = 0; + httpDebugFlag = 0; + mqttDebugFlag = 0; + monDebugFlag = 0; + uDebugFlag = 0; + rpcDebugFlag = 0; + odbcDebugFlag = 0; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + cqDebugFlag = 0; + debugFlag = 0; + char temp[PATH_MAX]; snprintf(temp, PATH_MAX, "%s/taosdlog", path); if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { @@ -32,7 +51,7 @@ void* runServer(void* param) { } } -void initOption(SDnodeOpt* pOption, char* path, char* fqdn, uint16_t port) { +void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t port) { pOption->sver = 1; pOption->numOfCores = 1; pOption->numOfSupportMnodes = 1; @@ -46,15 +65,16 @@ void initOption(SDnodeOpt* pOption, char* path, char* fqdn, uint16_t port) { pOption->shellActivityTimer = 30; pOption->serverPort = port; strcpy(pOption->dataDir, path); - snprintf(pOption->localEp, TSDB_EP_LEN, "%s:&u", fqdn, port); + snprintf(pOption->localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); snprintf(pOption->localFqdn, TSDB_FQDN_LEN, "%s", fqdn); - snprintf(pOption->firstEp, TSDB_EP_LEN, "%s:&u", fqdn, port); - - taosRemoveDir(path); - taosMkDir(path); + snprintf(pOption->firstEp, TSDB_EP_LEN, "%s:%u", fqdn, port); } -SServer* createServer(char* path, char* fqdn, uint16_t port) { +SServer* createServer(const char* path, const char* fqdn, uint16_t port) { + taosRemoveDir(path); + taosMkDir(path); + initLog(path); + SDnodeOpt option = {0}; initOption(&option, path, fqdn, port); @@ -80,11 +100,11 @@ void dropServer(SServer* pServer) { void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SClient* pClient = (SClient*)parent; pClient->pRsp = pMsg; - //taosMsleep(1000000); + // taosMsleep(1000000); tsem_post(&pClient->sem); } -SClient* createClient(char *user, char *pass) { +SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port) { SClient* pClient = (SClient*)calloc(1, sizeof(SClient)); ASSERT(pClient); @@ -99,7 +119,7 @@ SClient* createClient(char *user, char *pass) { rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = 30 * 1000; - rpcInit.user = user; + rpcInit.user = (char*)user; rpcInit.ckey = "key"; rpcInit.parent = pClient; rpcInit.secret = (char*)secretEncrypt; @@ -110,6 +130,8 @@ SClient* createClient(char *user, char *pass) { ASSERT(pClient->clientRpc); tsem_init(&pClient->sem, 0, 0); + strcpy(pClient->fqdn, fqdn); + pClient->port = port; return pClient; } @@ -123,8 +145,8 @@ void sendMsg(SClient* pClient, SRpcMsg* pMsg) { SEpSet epSet = {0}; epSet.inUse = 0; epSet.numOfEps = 1; - epSet.port[0] = 9527; - strcpy(epSet.fqdn[0], "localhost"); + epSet.port[0] = pClient->port; + strcpy(epSet.fqdn[0], pClient->fqdn); rpcSendRequest(pClient->clientRpc, &epSet, pMsg, NULL); tsem_wait(&pClient->sem); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.h b/source/dnode/mgmt/impl/test/sut/deploy.h index b830c7351e..8a352b96e8 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.h +++ b/source/dnode/mgmt/impl/test/sut/deploy.h @@ -31,13 +31,15 @@ typedef struct { } SServer; typedef struct { + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; void* clientRpc; SRpcMsg* pRsp; tsem_t sem; } SClient; -SServer* createServer(char* path, char *fqdn, uint16_t port); +SServer* createServer(const char* path, const char* fqdn, uint16_t port); void dropServer(SServer* pServer); -SClient* createClient(char *user, char *pass); +SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port); void dropClient(SClient* pClient); void sendMsg(SClient* pClient, SRpcMsg* pMsg); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index aaf86c15b6..6ad0f05c2a 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -296,7 +296,6 @@ typedef struct SShowObj { void *pIter; void *pVgIter; SMnode *pMnode; - SShowObj **ppShow; char db[TSDB_FULL_DB_NAME_LEN]; int16_t offset[TSDB_MAX_COLUMNS]; int32_t bytes[TSDB_MAX_COLUMNS]; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index ac606b2ff6..6d95c5ffb9 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -172,9 +172,9 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId, char *newUser, u } static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - if (pConn == NULL) return; + + SProfileMgmt *pMgmt = &pMnode->profileMgmt; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); } @@ -316,6 +316,11 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { } else { mDebug("user:%s, conn:%d is freed and create a new conn:%d", pMsg->user, pReq->connId, pConn->connId); } + } else if (pConn->killed) { + mDebug("user:%s, conn:%d is already killed", pMsg->user, pReq->connId, pConn->connId); + terrno = TSDB_CODE_TSC_INVALID_CONNECTION; + return -1; + } else { } SHeartBeatRsp *pRsp = rpcMallocCont(sizeof(SHeartBeatRsp)); @@ -368,7 +373,7 @@ static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; int32_t connId = htonl(pKill->connId); int32_t queryId = htonl(pKill->queryId); - mInfo("kill query msg is received, queryId:%s", pKill->queryId); + mInfo("kill query msg is received, queryId:%d", pKill->queryId); SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); if (pConn == NULL) { @@ -399,7 +404,7 @@ static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg) { SKillStreamMsg *pKill = pMsg->rpcMsg.pCont; int32_t connId = htonl(pKill->connId); int32_t streamId = htonl(pKill->streamId); - mDebug("kill stream msg is received, streamId:%s", streamId); + mDebug("kill stream msg is received, streamId:%d", streamId); SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); if (pConn == NULL) { @@ -432,11 +437,11 @@ static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); if (pConn == NULL) { - mError("connId:%s, failed to kill connection, conn not exist", connId); + mError("connId:%d, failed to kill connection, conn not exist", connId); terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%s, is killed by user:%s", connId, pMsg->user); + mInfo("connId:%d, is killed by user:%s", connId, pMsg->user); pConn->killed = 1; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); return TSDB_CODE_SUCCESS; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index e652f94957..1fe6684a01 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -16,19 +16,19 @@ #define _DEFAULT_SOURCE #include "mndShow.h" -static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg); -static int32_t mndProcessRetrieveMsg( SMnodeMsg *pMsg); -static bool mndCheckRetrieveFinished(SShowObj *pShow); -static int32_t mndAcquireShowObj(SMnode *pMnode, SShowObj *pShow); -static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); -static int32_t mndPutShowObj(SMnode *pMnode, SShowObj *pShow); -static void mndFreeShowObj(void *ppShow); -static char *mndShowStr(int32_t showType); +static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg); +static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMsg); +static bool mndCheckRetrieveFinished(SShowObj *pShow); +static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg); +static void mndFreeShowObj(SShowObj *pShow); +static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId); +static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); +static char *mndShowStr(int32_t showType); int32_t mndInitShow(SMnode *pMnode) { SShowMgmt *pMgmt = &pMnode->showMgmt; - pMgmt->cache = taosCacheInit(TSDB_CACHE_PTR_KEY, 5, true, mndFreeShowObj, "show"); + pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5, true, (__cache_free_fn_t)mndFreeShowObj, "show"); if (pMgmt->cache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to alloc show cache since %s", terrstr()); @@ -48,47 +48,41 @@ void mndCleanupShow(SMnode *pMnode) { } } -static int32_t mndAcquireShowObj(SMnode *pMnode, SShowObj *pShow) { - TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pShow; - +static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg) { SShowMgmt *pMgmt = &pMnode->showMgmt; - SShowObj **ppShow = taosCacheAcquireByKey(pMgmt->cache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE)); - if (ppShow) { - mTrace("show:%d, data:%p acquired from cache", pShow->id, ppShow); - return 0; - } - return -1; -} + int32_t showId = atomic_add_fetch_32(&pMgmt->showId, 1); + if (showId == 0) atomic_add_fetch_32(&pMgmt->showId, 1); -static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { - SMnode *pMnode = pShow->pMnode; - SShowMgmt *pMgmt = &pMnode->showMgmt; - SShowObj **ppShow = (SShowObj **)pShow->ppShow; - taosCacheRelease(pMgmt->cache, (void **)(&ppShow), forceRemove); - mDebug("show:%d, data:%p released from cache, force:%d", pShow->id, ppShow, forceRemove); -} - -static int32_t mndPutShowObj(SMnode *pMnode, SShowObj *pShow) { - SShowMgmt *pMgmt = &pMnode->showMgmt; - int32_t lifeSpan = pMnode->shellActivityTimer * 6 * 1000; - - TSDB_CACHE_PTR_TYPE val = (TSDB_CACHE_PTR_TYPE)pShow; - pShow->id = atomic_add_fetch_32(&pMgmt->showId, 1); - SShowObj **ppShow = - taosCachePut(pMgmt->cache, &val, sizeof(TSDB_CACHE_PTR_TYPE), &pShow, sizeof(TSDB_CACHE_PTR_TYPE), lifeSpan); - if (ppShow == NULL) { + int32_t size = sizeof(SShowObj) + pMsg->payloadLen; + SShowObj *pShow = calloc(1, size); + if (pShow != NULL) { + pShow->id = showId; + pShow->pMnode = pMnode; + pShow->type = pMsg->type; + pShow->payloadLen = pMsg->payloadLen; + memcpy(pShow->db, pMsg->db, TSDB_FULL_DB_NAME_LEN); + memcpy(pShow->payload, pMsg->payload, pMsg->payloadLen); + } else { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:%d, failed to put into cache", pShow->id); - return -1; + mError("failed to process show-meta msg:%s since %s", mndShowStr(pMsg->type), terrstr()); + return NULL; } - mTrace("show:%d, data:%p put into cache", pShow->id, ppShow); - return 0; + int32_t keepTime = pMnode->shellActivityTimer * 6 * 1000; + SShowObj *pShowRet = taosCachePut(pMgmt->cache, &showId, sizeof(int32_t), pShow, size, keepTime); + free(pShow); + if (pShowRet == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("show:%d, failed to put into cache since %s", showId, terrstr()); + return NULL; + } else { + mTrace("show:%d, data:%p created", showId, pShowRet); + return pShowRet; + } } -static void mndFreeShowObj(void *ppShow) { - SShowObj *pShow = *(SShowObj **)ppShow; +static void mndFreeShowObj(SShowObj *pShow) { SMnode *pMnode = pShow->pMnode; SShowMgmt *pMgmt = &pMnode->showMgmt; @@ -103,8 +97,29 @@ static void mndFreeShowObj(void *ppShow) { } } - mDebug("show:%d, data:%p destroyed", pShow->id, ppShow); - tfree(pShow); + mTrace("show:%d, data:%p destroyed", pShow->id, pShow); +} + +static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + + SShowObj *pShow = taosCacheAcquireByKey(pMgmt->cache, &showId, sizeof(int32_t)); + if (pShow == NULL) { + mError("show:%d, already destroyed", showId); + return NULL; + } + + mTrace("show:%d, data:%p acquired from cache", pShow->id, pShow); + return pShow; +} + +static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { + if (pShow == NULL) return; + mTrace("show:%d, data:%p released from cache, force:%d", pShow->id, pShow, forceRemove); + + SMnode *pMnode = pShow->pMnode; + SShowMgmt *pMgmt = &pMnode->showMgmt; + taosCacheRelease(pMgmt->cache, (void **)(&pShow), forceRemove); } static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { @@ -112,7 +127,7 @@ static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { SShowMgmt *pMgmt = &pMnode->showMgmt; SShowMsg *pMsg = pMnodeMsg->rpcMsg.pCont; int8_t type = pMsg->type; - uint16_t payloadLen = htonl(pMsg->payloadLen); + int16_t payloadLen = htonl(pMsg->payloadLen); if (type <= TSDB_MGMT_TABLE_START || type >= TSDB_MGMT_TABLE_MAX) { terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; @@ -127,27 +142,13 @@ static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { return -1; } - int32_t size = sizeof(SShowObj) + payloadLen; - SShowObj *pShow = calloc(1, size); - if (pShow != NULL) { - pShow->pMnode = pMnode; - pShow->type = type; - pShow->payloadLen = payloadLen; - memcpy(pShow->db, pMsg->db, TSDB_FULL_DB_NAME_LEN); - memcpy(pShow->payload, pMsg->payload, payloadLen); - } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; + SShowObj *pShow = mndCreateShowObj(pMnode, pMsg); + if (pShow == NULL) { mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr()); return -1; } - if (mndPutShowObj(pMnode, pShow) == 0) { - mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr()); - free(pShow); - return -1; - } - - size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; + int32_t size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; SShowRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { mndReleaseShowObj(pShow, true); @@ -156,15 +157,14 @@ static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { return -1; } - pRsp->qhandle = htobe64((uint64_t)pShow); - - int32_t code = (*metaFp)(pMnodeMsg,pShow, &pRsp->tableMeta); - mDebug("show:%d, type:%s, get meta finished, numOfRows:%d cols:%d result:%s", pShow->id, mndShowStr(type), - pShow->numOfRows, pShow->numOfColumns, tstrerror(code)); + int32_t code = (*metaFp)(pMnodeMsg, pShow, &pRsp->tableMeta); + mDebug("show:%d, data:%p get meta finished, numOfRows:%d cols:%d type:%s result:%s", pShow->id, pShow, + pShow->numOfRows, pShow->numOfColumns, mndShowStr(type), tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { pMnodeMsg->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; pMnodeMsg->pCont = pRsp; + pRsp->showId = htonl(pShow->id); mndReleaseShowObj(pShow, false); return TSDB_CODE_SUCCESS; } else { @@ -182,14 +182,10 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { int32_t rowsRead = 0; SRetrieveTableMsg *pRetrieve = pMnodeMsg->rpcMsg.pCont; - pRetrieve->qhandle = htobe64(pRetrieve->qhandle); - SShowObj *pShow = (SShowObj *)pRetrieve->qhandle; + int32_t showId = htonl(pRetrieve->showId); - /* - * in case of server restart, apps may hold qhandle created by server before - * restart, which is actually invalid, therefore, signature check is required. - */ - if (mndAcquireShowObj(pMnode, pShow) != 0) { + SShowObj *pShow = mndAcquireShowObj(pMnode, showId); + if (pShow == NULL) { terrno = TSDB_CODE_MND_INVALID_SHOWOBJ; mError("failed to process show-retrieve msg:%p since %s", pShow, terrstr()); return -1; @@ -199,15 +195,16 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { if (retrieveFp == NULL) { mndReleaseShowObj(pShow, false); terrno = TSDB_CODE_MSG_NOT_PROCESSED; - mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr()); + mError("show:%d, data:%p failed to retrieve data since %s", pShow->id, pShow, terrstr()); return -1; } - mDebug("show:%d, type:%s, start retrieve data, numOfReads:%d numOfRows:%d", pShow->id, mndShowStr(pShow->type), - pShow->numOfReads, pShow->numOfRows); + mDebug("show:%d, data:%p start retrieve data, numOfReads:%d numOfRows:%d type:%s", pShow->id, pShow, + pShow->numOfReads, pShow->numOfRows, mndShowStr(pShow->type)); if (mndCheckRetrieveFinished(pShow)) { - mDebug("show:%d, read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads, pShow->numOfRows); + mDebug("show:%d, data:%p read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow, pShow->numOfReads, + pShow->numOfRows); pShow->numOfReads = pShow->numOfRows; } @@ -230,7 +227,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { if (pRsp == NULL) { mndReleaseShowObj(pShow, false); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr()); + mError("show:%d, data:%p failed to retrieve data since %s", pShow->id, pShow, terrstr()); return -1; } @@ -239,20 +236,20 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { rowsRead = (*retrieveFp)(pMnodeMsg, pShow, pRsp->data, rowsToRead); } - mDebug("show:%d, stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); + mDebug("show:%d, data:%p stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, pShow, rowsRead, rowsToRead); pRsp->numOfRows = htonl(rowsRead); - pRsp->precision = (int16_t)htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision + pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision pMnodeMsg->pCont = pRsp; pMnodeMsg->contLen = size; if (rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; - mDebug("%p, retrieve completed", pShow); + mDebug("show:%d, data:%p retrieve completed", pShow->id, pShow); mndReleaseShowObj(pShow, true); } else { - mDebug("%p, retrieve not completed yet", pShow); + mDebug("show:%d, data:%p retrieve not completed yet", pShow->id, pShow); mndReleaseShowObj(pShow, false); } @@ -307,7 +304,7 @@ static char *mndShowStr(int32_t showType) { static bool mndCheckRetrieveFinished(SShowObj *pShow) { if (pShow->pIter == NULL && pShow->numOfReads != 0) { return true; - } + } return false; } From 81df0733794891c7fbf60476ca70daa9227834b5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 6 Dec 2021 14:44:27 +0800 Subject: [PATCH 11/16] [td-10564] fix compiler error. --- source/libs/executor/src/executorimpl.c | 416 ++++++++++++------------ 1 file changed, 209 insertions(+), 207 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d70b61bb4a..0999c44fab 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -289,7 +289,7 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeE return; } - int32_t orderId = pRuntimeEnv->pQueryAttr->order.orderColId; + int32_t orderId = pRuntimeEnv->pQueryAttr->order.col.info.colId; if (orderId <= 0) { return; } @@ -1914,7 +1914,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExpr pCtx->param[3].i = functionId; pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[1].i = pQueryAttr->order.orderColId; + pCtx->param[1].i = pQueryAttr->order.col.info.colId; } else if (functionId == FUNCTION_INTERP) { pCtx->param[2].i = (int8_t)pQueryAttr->fillType; if (pQueryAttr->fillVal != NULL) { @@ -2013,162 +2013,162 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf int32_t* op = taosArrayGet(pOperator, i); switch (*op) { - case OP_TagScan: { - pRuntimeEnv->proot = createTagScanOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - break; - } - case OP_MultiTableTimeInterval: { - pRuntimeEnv->proot = - createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - break; - } - case OP_AllMultiTableTimeInterval: { - pRuntimeEnv->proot = - createAllMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - break; - } - case OP_TimeWindow: { - pRuntimeEnv->proot = - createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput && opType != OP_Join) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_AllTimeWindow: { - pRuntimeEnv->proot = - createAllTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput && opType != OP_Join) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_Groupby: { - pRuntimeEnv->proot = - createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_SessionWindow: { - pRuntimeEnv->proot = - createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - case OP_MultiTableAggregate: { - pRuntimeEnv->proot = - createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - break; - } - case OP_Aggregate: { - pRuntimeEnv->proot = - createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput && opType != OP_Join) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - - case OP_Project: { // TODO refactor to remove arith operator. - SOperatorInfo* prev = pRuntimeEnv->proot; - if (i == 0) { - pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - if (pRuntimeEnv->proot != NULL && prev->operatorType != OP_DummyInput && prev->operatorType != OP_Join) { // TODO refactor - setTableScanFilterOperatorInfo(prev->info, pRuntimeEnv->proot); - } - } else { - prev = pRuntimeEnv->proot; - assert(pQueryAttr->pExpr2 != NULL); - pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); - } - break; - } - - case OP_StateWindow: { - pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; - if (opType != OP_DummyInput) { - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); - } - break; - } - - case OP_Limit: { - pRuntimeEnv->proot = createLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot); - break; - } - - case OP_Filter: { // todo refactor - int32_t numOfFilterCols = 0; - if (pQueryAttr->stableQuery) { - SColumnInfo* pColInfo = - extractColumnFilterInfo(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3, &numOfFilterCols); - pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, - pQueryAttr->numOfExpr3, pColInfo, numOfFilterCols); - freeColumnInfo(pColInfo, pQueryAttr->numOfExpr3); - } else { - SColumnInfo* pColInfo = - extractColumnFilterInfo(pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &numOfFilterCols); - pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, - pQueryAttr->numOfOutput, pColInfo, numOfFilterCols); - freeColumnInfo(pColInfo, pQueryAttr->numOfOutput); - } - - break; - } - - case OP_Fill: { - SOperatorInfo* pInfo = pRuntimeEnv->proot; - pRuntimeEnv->proot = createFillOperatorInfo(pRuntimeEnv, pInfo, pInfo->pExpr, pInfo->numOfOutput, pQueryAttr->multigroupResult); - break; - } - - case OP_MultiwayMergeSort: { - pRuntimeEnv->proot = createMultiwaySortOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, 4096, merger); - break; - } - - case OP_GlobalAggregate: { // If fill operator exists, the result rows of different group can not be in the same SSDataBlock. - bool multigroupResult = pQueryAttr->multigroupResult; - if (pQueryAttr->multigroupResult) { - multigroupResult = (pQueryAttr->fillType == TSDB_FILL_NONE); - } - - pRuntimeEnv->proot = createGlobalAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, - pQueryAttr->numOfExpr3, merger, pQueryAttr->pUdfInfo, multigroupResult); - break; - } - - case OP_SLimit: { - int32_t num = pRuntimeEnv->proot->numOfOutput; - SExprInfo* pExpr = pRuntimeEnv->proot->pExpr; - pRuntimeEnv->proot = createSLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pExpr, num, merger, pQueryAttr->multigroupResult); - break; - } - - case OP_Distinct: { - pRuntimeEnv->proot = createDistinctOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - break; - } - - case OP_Order: { - pRuntimeEnv->proot = createOrderOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order); - break; - } +// case OP_TagScan: { +// pRuntimeEnv->proot = createTagScanOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// break; +// } +// case OP_MultiTableTimeInterval: { +// pRuntimeEnv->proot = +// createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// break; +// } +// case OP_AllMultiTableTimeInterval: { +// pRuntimeEnv->proot = +// createAllMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// break; +// } +// case OP_TimeWindow: { +// pRuntimeEnv->proot = +// createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// if (opType != OP_DummyInput && opType != OP_Join) { +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// } +// break; +// } +// case OP_AllTimeWindow: { +// pRuntimeEnv->proot = +// createAllTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// if (opType != OP_DummyInput && opType != OP_Join) { +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// } +// break; +// } +// case OP_Groupby: { +// pRuntimeEnv->proot = +// createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// +// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// if (opType != OP_DummyInput) { +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// } +// break; +// } +// case OP_SessionWindow: { +// pRuntimeEnv->proot = +// createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// if (opType != OP_DummyInput) { +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// } +// break; +// } +// case OP_MultiTableAggregate: { +// pRuntimeEnv->proot = +// createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// break; +// } +// case OP_Aggregate: { +// pRuntimeEnv->proot = +// createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// +// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// if (opType != OP_DummyInput && opType != OP_Join) { +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// } +// break; +// } +// +// case OP_Project: { // TODO refactor to remove arith operator. +// SOperatorInfo* prev = pRuntimeEnv->proot; +// if (i == 0) { +// pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// if (pRuntimeEnv->proot != NULL && prev->operatorType != OP_DummyInput && prev->operatorType != OP_Join) { // TODO refactor +// setTableScanFilterOperatorInfo(prev->info, pRuntimeEnv->proot); +// } +// } else { +// prev = pRuntimeEnv->proot; +// assert(pQueryAttr->pExpr2 != NULL); +// pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); +// } +// break; +// } +// +// case OP_StateWindow: { +// pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// if (opType != OP_DummyInput) { +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// } +// break; +// } +// +// case OP_Limit: { +// pRuntimeEnv->proot = createLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot); +// break; +// } +// +// case OP_Filter: { // todo refactor +// int32_t numOfFilterCols = 0; +// if (pQueryAttr->stableQuery) { +// SColumnInfo* pColInfo = +// extractColumnFilterInfo(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3, &numOfFilterCols); +// pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, +// pQueryAttr->numOfExpr3, pColInfo, numOfFilterCols); +// freeColumnInfo(pColInfo, pQueryAttr->numOfExpr3); +// } else { +// SColumnInfo* pColInfo = +// extractColumnFilterInfo(pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &numOfFilterCols); +// pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, +// pQueryAttr->numOfOutput, pColInfo, numOfFilterCols); +// freeColumnInfo(pColInfo, pQueryAttr->numOfOutput); +// } +// +// break; +// } +// +// case OP_Fill: { +// SOperatorInfo* pInfo = pRuntimeEnv->proot; +// pRuntimeEnv->proot = createFillOperatorInfo(pRuntimeEnv, pInfo, pInfo->pExpr, pInfo->numOfOutput, pQueryAttr->multigroupResult); +// break; +// } +// +// case OP_MultiwayMergeSort: { +// pRuntimeEnv->proot = createMultiwaySortOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, 4096, merger); +// break; +// } +// +// case OP_GlobalAggregate: { // If fill operator exists, the result rows of different group can not be in the same SSDataBlock. +// bool multigroupResult = pQueryAttr->multigroupResult; +// if (pQueryAttr->multigroupResult) { +// multigroupResult = (pQueryAttr->fillType == TSDB_FILL_NONE); +// } +// +// pRuntimeEnv->proot = createGlobalAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, +// pQueryAttr->numOfExpr3, merger, pQueryAttr->pUdfInfo, multigroupResult); +// break; +// } +// +// case OP_SLimit: { +// int32_t num = pRuntimeEnv->proot->numOfOutput; +// SExprInfo* pExpr = pRuntimeEnv->proot->pExpr; +// pRuntimeEnv->proot = createSLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pExpr, num, merger, pQueryAttr->multigroupResult); +// break; +// } +// +// case OP_Distinct: { +// pRuntimeEnv->proot = createDistinctOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); +// break; +// } +// +// case OP_Order: { +// pRuntimeEnv->proot = createOrderOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order); +// break; +// } default: { assert(0); @@ -4557,22 +4557,22 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr setResultBufSize(pQueryAttr, &pRuntimeEnv->resultInfo); switch(tbScanner) { - case OP_TableBlockInfoScan: { - pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); - break; - } - case OP_TableSeqScan: { - pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); - break; - } - case OP_DataBlocksOptScan: { - pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); - break; - } - case OP_TableScan: { - pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); - break; - } +// case OP_TableBlockInfoScan: { +// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); +// break; +// } +// case OP_TableSeqScan: { +// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); +// break; +// } +// case OP_DataBlocksOptScan: { +// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); +// break; +// } +// case OP_TableScan: { +// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); +// break; +// } default: { // do nothing break; } @@ -4881,7 +4881,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableScanOperator"; - pOperator->operatorType = OP_TableScan; +// pOperator->operatorType = OP_TableScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -4905,7 +4905,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeE SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableSeqScanOperator"; - pOperator->operatorType = OP_TableSeqScan; +// pOperator->operatorType = OP_TableSeqScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -4930,7 +4930,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRu SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableBlockInfoScanOperator"; - pOperator->operatorType = OP_TableBlockInfoScan; +// pOperator->operatorType = OP_TableBlockInfoScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -4946,7 +4946,7 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf pTableScanInfo->pExpr = pDownstream->pExpr; // TODO refactor to use colId instead of pExpr pTableScanInfo->numOfOutput = pDownstream->numOfOutput; - +#if 0 if (pDownstream->operatorType == OP_Aggregate || pDownstream->operatorType == OP_MultiTableAggregate) { SAggOperatorInfo* pAggInfo = pDownstream->info; @@ -4995,6 +4995,8 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf } else { assert(0); } +#endif + } SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { @@ -5009,7 +5011,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); pOptr->name = "DataBlocksOptimizedScanOperator"; - pOptr->operatorType = OP_DataBlocksOptScan; +// pOptr->operatorType = OP_DataBlocksOptScan; pOptr->pRuntimeEnv = pRuntimeEnv; pOptr->blockingOptr = false; pOptr->info = pInfo; @@ -5161,7 +5163,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "GlobalAggregate"; - pOperator->operatorType = OP_GlobalAggregate; +// pOperator->operatorType = OP_GlobalAggregate; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -5205,7 +5207,7 @@ SOperatorInfo *createMultiwaySortOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SEx SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiwaySortOperator"; - pOperator->operatorType = OP_MultiwayMergeSort; +// pOperator->operatorType = OP_MultiwayMergeSort; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -5312,7 +5314,7 @@ SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "InMemoryOrder"; - pOperator->operatorType = OP_Order; +// pOperator->operatorType = OP_Order; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -5358,10 +5360,10 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); } - if (upstream->operatorType == OP_DataBlocksOptScan) { - STableScanInfo* pScanInfo = upstream->info; - order = getTableScanOrder(pScanInfo); - } +// if (upstream->operatorType == OP_DataBlocksOptScan) { +// STableScanInfo* pScanInfo = upstream->info; +// order = getTableScanOrder(pScanInfo); +// } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); @@ -5413,10 +5415,10 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - if (upstream->operatorType == OP_DataBlocksOptScan) { - STableScanInfo* pScanInfo = upstream->info; - order = getTableScanOrder(pScanInfo); - } +// if (upstream->operatorType == OP_DataBlocksOptScan) { +// STableScanInfo* pScanInfo = upstream->info; +// order = getTableScanOrder(pScanInfo); +// } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); @@ -6268,7 +6270,7 @@ SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpera SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableAggregate"; - pOperator->operatorType = OP_Aggregate; +// pOperator->operatorType = OP_Aggregate; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -6363,7 +6365,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SO SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableAggregate"; - pOperator->operatorType = OP_MultiTableAggregate; +// pOperator->operatorType = OP_MultiTableAggregate; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -6393,7 +6395,7 @@ SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "ProjectOperator"; - pOperator->operatorType = OP_Project; +// pOperator->operatorType = OP_Project; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -6452,7 +6454,7 @@ SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "FilterOperator"; - pOperator->operatorType = OP_Filter; +// pOperator->operatorType = OP_Filter; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->numOfOutput = numOfOutput; @@ -6473,7 +6475,7 @@ SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "LimitOperator"; - pOperator->operatorType = OP_Limit; +// pOperator->operatorType = OP_Limit; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->exec = doLimit; @@ -6494,7 +6496,7 @@ SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOp SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = OP_TimeWindow; +// pOperator->operatorType = OP_TimeWindow; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->pExpr = pExpr; @@ -6519,7 +6521,7 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "AllTimeIntervalAggOperator"; - pOperator->operatorType = OP_AllTimeWindow; +// pOperator->operatorType = OP_AllTimeWindow; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->pExpr = pExpr; @@ -6543,7 +6545,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpe SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "StateWindowOperator"; - pOperator->operatorType = OP_StateWindow; +// pOperator->operatorType = OP_StateWindow; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->pExpr = pExpr; @@ -6568,7 +6570,7 @@ SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "SessionWindowAggOperator"; - pOperator->operatorType = OP_SessionWindow; +// pOperator->operatorType = OP_SessionWindow; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->pExpr = pExpr; @@ -6591,7 +6593,7 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRunti SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableTimeIntervalOperator"; - pOperator->operatorType = OP_MultiTableTimeInterval; +// pOperator->operatorType = OP_MultiTableTimeInterval; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->pExpr = pExpr; @@ -6615,7 +6617,7 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRu SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "AllMultiTableTimeIntervalOperator"; - pOperator->operatorType = OP_AllMultiTableTimeInterval; +// pOperator->operatorType = OP_AllMultiTableTimeInterval; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->pExpr = pExpr; @@ -6651,7 +6653,7 @@ SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pOperator->name = "GroupbyAggOperator"; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; - pOperator->operatorType = OP_Groupby; +// pOperator->operatorType = OP_Groupby; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; @@ -6690,7 +6692,7 @@ SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->name = "FillOperator"; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; - pOperator->operatorType = OP_Fill; +// pOperator->operatorType = OP_Fill; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; @@ -6738,7 +6740,7 @@ SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "SLimitOperator"; - pOperator->operatorType = OP_SLimit; +// pOperator->operatorType = OP_SLimit; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->exec = doSLimit; @@ -6894,7 +6896,7 @@ SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInf SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "SeqTableTagScan"; - pOperator->operatorType = OP_TagScan; +// pOperator->operatorType = OP_TagScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -7035,7 +7037,7 @@ SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperat pOperator->name = "DistinctOperator"; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; - pOperator->operatorType = OP_Distinct; +// pOperator->operatorType = OP_Distinct; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; @@ -8034,7 +8036,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S pQueryAttr->limit.limit = pQueryMsg->limit; pQueryAttr->limit.offset = pQueryMsg->offset; pQueryAttr->order.order = pQueryMsg->order; - pQueryAttr->order.orderColId = pQueryMsg->orderColId; + pQueryAttr->order.col.info.colId = pQueryMsg->orderColId; pQueryAttr->pExpr1 = pExprs; pQueryAttr->pExpr2 = pSecExprs; pQueryAttr->numOfExpr2 = pQueryMsg->secondStageOutput; From 8d5f5901c4903e83ba5e945e26e0fd7430b8552f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 6 Dec 2021 15:06:29 +0800 Subject: [PATCH 12/16] TD-10431 profile test --- include/common/taosmsg.h | 85 +++---- .../dnode/mgmt/impl/test/profile/profile.cpp | 217 +++++++++++++++++- source/dnode/mnode/impl/src/mndProfile.c | 169 ++++++++------ source/dnode/mnode/impl/src/mndShow.c | 11 +- 4 files changed, 361 insertions(+), 121 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 9443ef7475..300de7b698 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -418,10 +418,10 @@ typedef struct { } SDropSTableMsg; typedef struct SColIndex { - int16_t colId; // column id - int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag - uint16_t flag; // denote if it is a tag or a normal column - char name[TSDB_COL_NAME_LEN + TSDB_DB_NAME_LEN + 1]; + int16_t colId; // column id + int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag + int16_t flag; // denote if it is a tag or a normal column + char name[TSDB_COL_NAME_LEN + TSDB_DB_NAME_LEN + 1]; } SColIndex; typedef struct SColumnFilterInfo { @@ -515,8 +515,8 @@ typedef struct { int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; // SSessionWindow sw; // session window - uint16_t tagCondLen; // tag length in current query - uint16_t colCondLen; // column length in current query + int16_t tagCondLen; // tag length in current query + int16_t colCondLen; // column length in current query int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx @@ -524,10 +524,10 @@ typedef struct { int16_t prjOrder; // global order in super table projection query. int64_t limit; int64_t offset; - uint32_t queryType; // denote another query process + int32_t queryType; // denote another query process int16_t numOfOutput; // final output columns numbers int16_t fillType; // interpolate type - uint64_t fillVal; // default value array list + int64_t fillVal; // default value array list int32_t secondStageOutput; STsBufInfo tsBuf; // tsBuf info int32_t numOfTags; // number of tags columns involved @@ -542,8 +542,11 @@ typedef struct { } SQueryTableMsg; typedef struct { - int32_t code; - union{uint64_t qhandle; uint64_t qId;}; // query handle + int32_t code; + union { + uint64_t qhandle; + uint64_t qId; + }; // query handle } SQueryTableRsp; // todo: the show handle should be replaced with id @@ -706,7 +709,7 @@ typedef struct { typedef struct { char db[TSDB_FULL_DB_NAME_LEN]; - uint32_t vgId; + int32_t vgId; int32_t cacheBlockSize; int32_t totalBlocks; int32_t daysPerFile; @@ -784,14 +787,14 @@ typedef struct STableMetaMsg { } STableMetaMsg; typedef struct SMultiTableMeta { - int32_t numOfTables; - int32_t numOfVgroup; - int32_t numOfUdf; - int32_t contLen; - uint8_t compressed; // denote if compressed or not - uint32_t rawLen; // size before compress - uint8_t metaClone; // make meta clone after retrieve meta from mnode - char meta[]; + int32_t numOfTables; + int32_t numOfVgroup; + int32_t numOfUdf; + int32_t contLen; + int8_t compressed; // denote if compressed or not + int32_t rawLen; // size before compress + uint8_t metaClone; // make meta clone after retrieve meta from mnode + char meta[]; } SMultiTableMeta; typedef struct { @@ -841,8 +844,8 @@ typedef struct { } SConfigTableMsg; typedef struct { - uint32_t dnodeId; - int32_t vgId; + int32_t dnodeId; + int32_t vgId; } SConfigVnodeMsg; typedef struct { @@ -851,29 +854,29 @@ typedef struct { } SCfgDnodeMsg; typedef struct { - char sql[TSDB_SHOW_SQL_LEN]; - uint32_t queryId; - int64_t useconds; - int64_t stime; - uint64_t qId; - uint64_t sqlObjId; - int32_t pid; - char fqdn[TSDB_FQDN_LEN]; - uint8_t stableQuery; - int32_t numOfSub; - char subSqlInfo[TSDB_SHOW_SUBQUERY_LEN]; //include subqueries' index, Obj IDs and states(C-complete/I-imcomplete) + char sql[TSDB_SHOW_SQL_LEN]; + int32_t queryId; + int64_t useconds; + int64_t stime; + int64_t qId; + int64_t sqlObjId; + int32_t pid; + char fqdn[TSDB_FQDN_LEN]; + int8_t stableQuery; + int32_t numOfSub; + char subSqlInfo[TSDB_SHOW_SUBQUERY_LEN]; // include subqueries' index, Obj IDs and states(C-complete/I-imcomplete) } SQueryDesc; typedef struct { - char sql[TSDB_SHOW_SQL_LEN]; - char dstTable[TSDB_TABLE_NAME_LEN]; - uint32_t streamId; - int64_t num; // number of computing/cycles - int64_t useconds; - int64_t ctime; - int64_t stime; - int64_t slidingTime; - int64_t interval; + char sql[TSDB_SHOW_SQL_LEN]; + char dstTable[TSDB_TABLE_NAME_LEN]; + int32_t streamId; + int64_t num; // number of computing/cycles + int64_t useconds; + int64_t ctime; + int64_t stime; + int64_t slidingTime; + int64_t interval; } SStreamDesc; typedef struct { diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 210885bd27..83636d1477 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -215,7 +215,7 @@ TEST_F(DndTestProfile, SConnectMsg_03) { sendMsg(pClient, &rpcMsg); SRpcMsg* pMsg = pClient->pRsp; ASSERT_NE(pMsg, nullptr); - ASSERT_NE(pMsg->code, 0); + ASSERT_EQ(pMsg->code, 0); SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); @@ -310,7 +310,7 @@ TEST_F(DndTestProfile, SKillConnMsg_01) { sendMsg(pClient, &rpcMsg); SRpcMsg* pMsg = pClient->pRsp; ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_TSC_INVALID_CONNECTION); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONNECTION); ASSERT_EQ(pMsg->contLen, 0); } @@ -338,7 +338,7 @@ TEST_F(DndTestProfile, SKillConnMsg_01) { EXPECT_EQ(pRsp->acctId, 1); EXPECT_GT(pRsp->clusterId, 0); - EXPECT_EQ(pRsp->superAuth, 1); + EXPECT_GT(pRsp->connId, connId); EXPECT_EQ(pRsp->readAuth, 1); EXPECT_EQ(pRsp->writeAuth, 1); @@ -446,6 +446,115 @@ TEST_F(DndTestProfile, SKillQueryMsg_02) { ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); } +TEST_F(DndTestProfile, SKillQueryMsg_03) { + ASSERT_NE(pClient, nullptr); + int32_t showId = 0; + + { + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pReq->type = TSDB_MGMT_TABLE_QUERIES; + strcpy(pReq->db, ""); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SShowMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + + SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->showId = htonl(pRsp->showId); + STableMetaMsg* pMeta = &pRsp->tableMeta; + pMeta->contLen = htonl(pMeta->contLen); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tid = htonl(pMeta->tid); + pMeta->uid = htobe64(pMeta->uid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pRsp->showId; + + EXPECT_NE(pRsp->showId, 0); + EXPECT_EQ(pMeta->contLen, 0); + EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->numOfColumns, 14); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tid, 0); + EXPECT_EQ(pMeta->uid, 0); + EXPECT_STREQ(pMeta->sTableName, ""); + EXPECT_EQ(pMeta->suid, 0); + + SSchema* pSchema = NULL; + pSchema = &pMeta->schema[0]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); + EXPECT_EQ(pSchema->bytes, 4); + EXPECT_STREQ(pSchema->name, "queryId"); + + pSchema = &pMeta->schema[1]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); + EXPECT_EQ(pSchema->bytes, 4); + EXPECT_STREQ(pSchema->name, "connId"); + + pSchema = &pMeta->schema[2]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "user"); + + pSchema = &pMeta->schema[3]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "ip:port"); + } + + { + SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pReq->showId = htonl(showId); + pReq->free = 0; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SRetrieveTableMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->offset = htobe64(pRsp->offset); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->compLen = htonl(pRsp->compLen); + + EXPECT_EQ(pRsp->numOfRows, 0); + EXPECT_EQ(pRsp->offset, 0); + EXPECT_EQ(pRsp->useconds, 0); + EXPECT_EQ(pRsp->completed, 1); + EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRsp->compressed, 0); + EXPECT_EQ(pRsp->reserved, 0); + EXPECT_EQ(pRsp->compLen, 0); + } +} + TEST_F(DndTestProfile, SKillStreamMsg_01) { ASSERT_NE(pClient, nullptr); @@ -523,3 +632,105 @@ TEST_F(DndTestProfile, SKillStreamMsg_02) { ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); } + +TEST_F(DndTestProfile, SKillStreamMsg_03) { + ASSERT_NE(pClient, nullptr); + int32_t showId = 0; + + { + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pReq->type = TSDB_MGMT_TABLE_STREAMS; + strcpy(pReq->db, ""); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SShowMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + + SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->showId = htonl(pRsp->showId); + STableMetaMsg* pMeta = &pRsp->tableMeta; + pMeta->contLen = htonl(pMeta->contLen); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tid = htonl(pMeta->tid); + pMeta->uid = htobe64(pMeta->uid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pRsp->showId; + + EXPECT_NE(pRsp->showId, 0); + EXPECT_EQ(pMeta->contLen, 0); + EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->numOfColumns, 10); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tid, 0); + EXPECT_EQ(pMeta->uid, 0); + EXPECT_STREQ(pMeta->sTableName, ""); + EXPECT_EQ(pMeta->suid, 0); + + SSchema* pSchema = NULL; + pSchema = &pMeta->schema[0]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); + EXPECT_EQ(pSchema->bytes, 4); + EXPECT_STREQ(pSchema->name, "streamId"); + + pSchema = &pMeta->schema[1]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); + EXPECT_EQ(pSchema->bytes, 4); + EXPECT_STREQ(pSchema->name, "connId"); + + pSchema = &pMeta->schema[2]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "user"); + } + + { + SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pReq->showId = htonl(showId); + pReq->free = 0; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SRetrieveTableMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->offset = htobe64(pRsp->offset); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->compLen = htonl(pRsp->compLen); + + EXPECT_EQ(pRsp->numOfRows, 0); + EXPECT_EQ(pRsp->offset, 0); + EXPECT_EQ(pRsp->useconds, 0); + EXPECT_EQ(pRsp->completed, 1); + EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRsp->compressed, 0); + EXPECT_EQ(pRsp->reserved, 0); + EXPECT_EQ(pRsp->compLen, 0); + } +} diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 6d95c5ffb9..8f51b1a0be 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -29,7 +29,7 @@ typedef struct { char user[TSDB_USER_LEN]; char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc int32_t pid; // pid of app that invokes taosc - int32_t connId; + int32_t id; int8_t killed; int8_t align; uint16_t port; @@ -46,7 +46,7 @@ typedef struct { static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app); static void mndFreeConn(SConnObj *pConn); -static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId, char *user, uint32_t ip, uint16_t port); +static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); @@ -57,11 +57,11 @@ static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg); static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg); static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); -static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow); +static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); -static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow); -static int32_t mndRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, SMnodeMsg *pMsg); +static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveStreams(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { @@ -109,7 +109,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t if (connId == 0) atomic_add_fetch_32(&pMgmt->connId, 1); SConnObj connObj = {.pid = pid, - .connId = connId, + .id = connId, .killed = 0, .port = port, .ip = ip, @@ -130,10 +130,10 @@ static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); if (pConn == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("conn:%d, failed to put into cache since %s, user:%s", connId, user, terrstr()); + mError("conn:%d, data:%p failed to put into cache since %s, user:%s", connId, pConn, user, terrstr()); return NULL; } else { - mDebug("conn:%d, is created, user:%s", connId, user); + mTrace("conn:%d, data:%p created, user:%s", pConn->id, pConn, user); return pConn; } } @@ -141,39 +141,29 @@ static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t static void mndFreeConn(SConnObj *pConn) { tfree(pConn->pQueries); tfree(pConn->pStreams); - mDebug("conn:%d, is destroyed", pConn->connId); + mTrace("conn:%d, data:%p destroyed", pConn->id, pConn); } -static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId, char *newUser, uint32_t newIp, uint16_t newPort) { +static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); if (pConn == NULL) { - mDebug("conn:%d, already destroyed, user:%s", connId, newUser); - return NULL; - } - - if (pConn->ip != newIp || pConn->port != newPort /* || strcmp(pConn->user, newUser) != 0 */) { - char oldIpStr[30]; - char newIpStr[30]; - taosIp2String(pConn->ip, oldIpStr); - taosIp2String(newIp, newIpStr); - mDebug("conn:%d, incoming conn user:%s ip:%s:%u, not match exist user:%s ip:%s:%u", connId, newUser, newIpStr, - newPort, pConn->user, oldIpStr, pConn->port); - - if (pMgmt->connId < connId) pMgmt->connId = connId + 1; - taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + mDebug("conn:%d, already destroyed", connId); return NULL; } int32_t keepTime = pMnode->shellActivityTimer * 3; pConn->lastAccess = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); + + mTrace("conn:%d, data:%p acquired from cache", pConn->id, pConn); return pConn; } static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { if (pConn == NULL) return; - + mTrace("conn:%d, data:%p released from cache", pConn->id, pConn); + SProfileMgmt *pMgmt = &pMnode->profileMgmt; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); } @@ -250,13 +240,13 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { } pRsp->clusterId = htonl(pMnode->clusterId); - pRsp->connId = htonl(pConn->connId); + pRsp->connId = htonl(pConn->id); mndGetMnodeEpSet(pMnode, &pRsp->epSet); mndReleaseConn(pMnode, pConn); pMsg->contLen = sizeof(SConnectRsp); pMsg->pCont = pRsp; - mDebug("user:%s, login from %s, conn:%d", info.user, ip, pConn->connId); + mDebug("user:%s, login from %s, conn:%d", info.user, ip, pConn->id); return 0; } @@ -296,7 +286,9 @@ static int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { } static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + SHeartBeatMsg *pReq = pMsg->rpcMsg.pCont; pReq->connId = htonl(pReq->connId); pReq->pid = htonl(pReq->pid); @@ -307,20 +299,33 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { return -1; } - SConnObj *pConn = mndAcquireConn(pMnode, pReq->connId, info.user, info.clientIp, info.clientPort); + SConnObj *pConn = mndAcquireConn(pMnode, pReq->connId); if (pConn == NULL) { pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); if (pConn == NULL) { mError("user:%s, conn:%d is freed and failed to create new conn since %s", pMsg->user, pReq->connId, terrstr()); return -1; } else { - mDebug("user:%s, conn:%d is freed and create a new conn:%d", pMsg->user, pReq->connId, pConn->connId); + mDebug("user:%s, conn:%d is freed and create a new conn:%d", pMsg->user, pReq->connId, pConn->id); } } else if (pConn->killed) { - mDebug("user:%s, conn:%d is already killed", pMsg->user, pReq->connId, pConn->connId); - terrno = TSDB_CODE_TSC_INVALID_CONNECTION; + mError("user:%s, conn:%d is already killed", pMsg->user, pConn->id); + terrno = TSDB_CODE_MND_INVALID_CONNECTION; return -1; } else { + if (pConn->ip != info.clientIp || pConn->port != info.clientPort /* || strcmp(pConn->user, info.user) != 0 */) { + char oldIpStr[40]; + char newIpStr[40]; + taosIpPort2String(pConn->ip, pConn->port, oldIpStr); + taosIpPort2String(info.clientIp, info.clientPort, newIpStr); + mError("conn:%d, incoming conn user:%s ip:%s, not match exist user:%s ip:%s", pConn->id, info.user, newIpStr, + pConn->user, oldIpStr); + + if (pMgmt->connId < pConn->id) pMgmt->connId = pConn->id + 1; + taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + terrno = TSDB_CODE_MND_INVALID_CONNECTION; + return -1; + } } SHeartBeatRsp *pRsp = rpcMallocCont(sizeof(SHeartBeatRsp)); @@ -346,7 +351,7 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { pConn->queryId = 0; } - pRsp->connId = htonl(pConn->connId); + pRsp->connId = htonl(pConn->id); pRsp->totalDnodes = htonl(1); pRsp->onlineDnodes = htonl(1); mndGetMnodeEpSet(pMnode, &pRsp->epSet); @@ -525,47 +530,47 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { SMnode *pMnode = pMsg->pMnode; int32_t numOfRows = 0; - SConnObj *pConnObj = NULL; + SConnObj *pConn = NULL; int32_t cols = 0; char *pWrite; char ipStr[TSDB_IPv4ADDR_LEN + 6]; while (numOfRows < rows) { - pShow->pIter = mndGetNextConn(pMnode, pShow->pIter, &pConnObj); - if (pConnObj == NULL) break; + pShow->pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); + if (pConn == NULL) break; cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pConnObj->connId; + *(int32_t *)pWrite = pConn->id; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->user, pShow->bytes[cols]); cols++; // app name pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->app, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->app, pShow->bytes[cols]); cols++; // app pid pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pConnObj->pid; + *(int32_t *)pWrite = pConn->pid; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - taosIpPort2String(pConnObj->ip, pConnObj->port, ipStr); + taosIpPort2String(pConn->ip, pConn->port, ipStr); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pConnObj->stime; + *(int64_t *)pWrite = pConn->stime; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - if (pConnObj->lastAccess < pConnObj->stime) pConnObj->lastAccess = pConnObj->stime; - *(int64_t *)pWrite = pConnObj->lastAccess; + if (pConn->lastAccess < pConn->stime) pConn->lastAccess = pConn->stime; + *(int64_t *)pWrite = pConn->lastAccess; cols++; numOfRows++; @@ -576,7 +581,7 @@ static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in return numOfRows; } -static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow) { +static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { SMnode *pMnode = pMsg->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -592,9 +597,15 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj * int32_t cols = 0; SSchema *pSchema = pMeta->schema; - pShow->bytes[cols] = QUERY_ID_SIZE + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "query_id"); + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "queryId"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "connId"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -687,40 +698,43 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj * static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { SMnode *pMnode = pMsg->pMnode; int32_t numOfRows = 0; - SConnObj *pConnObj = NULL; + SConnObj *pConn = NULL; int32_t cols = 0; char *pWrite; void *pIter; char str[TSDB_IPv4ADDR_LEN + 6] = {0}; while (numOfRows < rows) { - pIter = mndGetNextConn(pMnode, pShow->pIter, &pConnObj); - if (pConnObj == NULL) { + pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); + if (pConn == NULL) { pShow->pIter = pIter; break; } - if (numOfRows + pConnObj->numOfQueries >= rows) { + if (numOfRows + pConn->numOfQueries >= rows) { mndCancelGetNextConn(pMnode, pIter); break; } pShow->pIter = pIter; - for (int32_t i = 0; i < pConnObj->numOfQueries; ++i) { - SQueryDesc *pDesc = pConnObj->pQueries + i; + for (int32_t i = 0; i < pConn->numOfQueries; ++i) { + SQueryDesc *pDesc = pConn->pQueries + i; cols = 0; - snprintf(str, QUERY_ID_SIZE + 1, "%u:%u", pConnObj->connId, htonl(pDesc->queryId)); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]); + *(int64_t *)pWrite = htobe64(pDesc->queryId); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]); + *(int64_t *)pWrite = htobe64(pConn->id); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - snprintf(str, tListLen(str), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->user, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + snprintf(str, tListLen(str), "%s:%u", taosIpStr(pConn->ip), pConn->port); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]); cols++; @@ -749,7 +763,7 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, cols++; char epBuf[TSDB_EP_LEN + 1] = {0}; - snprintf(epBuf, tListLen(epBuf), "%s:%u", pDesc->fqdn, pConnObj->port); + snprintf(epBuf, tListLen(epBuf), "%s:%u", pDesc->fqdn, pConn->port); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; STR_WITH_MAXSIZE_TO_VARSTR(pWrite, epBuf, pShow->bytes[cols]); cols++; @@ -783,7 +797,7 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); } -static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow) { +static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { SMnode *pMnode = pMsg->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -799,12 +813,18 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj int32_t cols = 0; SSchema *pSchema = pMeta->schema; - pShow->bytes[cols] = QUERY_ID_SIZE + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "streamId"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "connId"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); @@ -867,39 +887,42 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj return 0; } -static int32_t mndRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, SMnodeMsg *pMsg) { +static int32_t mndRetrieveStreams(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { SMnode *pMnode = pMsg->pMnode; int32_t numOfRows = 0; - SConnObj *pConnObj = NULL; + SConnObj *pConn = NULL; int32_t cols = 0; char *pWrite; void *pIter; char ipStr[TSDB_IPv4ADDR_LEN + 6]; while (numOfRows < rows) { - pIter = mndGetNextConn(pMnode, pShow->pIter, &pConnObj); - if (pConnObj == NULL) { + pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); + if (pConn == NULL) { pShow->pIter = pIter; break; } - if (numOfRows + pConnObj->numOfStreams >= rows) { + if (numOfRows + pConn->numOfStreams >= rows) { mndCancelGetNextConn(pMnode, pIter); break; } pShow->pIter = pIter; - for (int32_t i = 0; i < pConnObj->numOfStreams; ++i) { - SStreamDesc *pDesc = pConnObj->pStreams + i; + for (int32_t i = 0; i < pConn->numOfStreams; ++i) { + SStreamDesc *pDesc = pConn->pStreams + i; cols = 0; - snprintf(ipStr, QUERY_ID_SIZE + 1, "%u:%u", pConnObj->connId, htonl(pDesc->streamId)); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); + *(int64_t *)pWrite = htobe64(pDesc->streamId); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]); + *(int64_t *)pWrite = htobe64(pConn->id); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->user, pShow->bytes[cols]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -907,7 +930,7 @@ static int32_t mndRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, SMn cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port); + snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConn->ip), pConn->port); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); cols++; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 1fe6684a01..ca0d9b9d50 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -16,14 +16,14 @@ #define _DEFAULT_SOURCE #include "mndShow.h" -static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg); -static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMsg); -static bool mndCheckRetrieveFinished(SShowObj *pShow); static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); static char *mndShowStr(int32_t showType); +static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg); +static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMsg); +static bool mndCheckRetrieveFinished(SShowObj *pShow); int32_t mndInitShow(SMnode *pMnode) { SShowMgmt *pMgmt = &pMnode->showMgmt; @@ -116,6 +116,9 @@ static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId) { static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { if (pShow == NULL) return; mTrace("show:%d, data:%p released from cache, force:%d", pShow->id, pShow, forceRemove); + + // A bug in tcache.c + forceRemove = 0; SMnode *pMnode = pShow->pMnode; SShowMgmt *pMgmt = &pMnode->showMgmt; @@ -244,7 +247,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { pMnodeMsg->pCont = pRsp; pMnodeMsg->contLen = size; - if (rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { + if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; mDebug("show:%d, data:%p retrieve completed", pShow->id, pShow); mndReleaseShowObj(pShow, true); From 88fbe65674f44faef157fe0a4bffed50e2298a7e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 6 Dec 2021 15:58:26 +0800 Subject: [PATCH 13/16] TD-10431 show test --- source/dnode/mgmt/impl/test/CMakeLists.txt | 3 +- .../dnode/mgmt/impl/test/profile/profile.cpp | 14 +- .../dnode/mgmt/impl/test/show/CMakeLists.txt | 29 +++ source/dnode/mgmt/impl/test/show/show.cpp | 238 ++++++++++++++++++ 4 files changed, 276 insertions(+), 8 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/show/CMakeLists.txt create mode 100644 source/dnode/mgmt/impl/test/show/show.cpp diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 79ae5631c2..84edabfc58 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1 +1,2 @@ -add_subdirectory(profile) \ No newline at end of file +add_subdirectory(profile) +add_subdirectory(show) \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 83636d1477..7c0e79cd69 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -51,7 +51,7 @@ TEST_F(DndTestProfile, SConnectMsg_01) { SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); pReq->pid = htonl(1234); - strcpy(pReq->app, "test01"); + strcpy(pReq->app, "dndTestProfile"); strcpy(pReq->db, ""); SRpcMsg rpcMsg = {0}; @@ -90,7 +90,7 @@ TEST_F(DndTestProfile, SConnectMsg_02) { SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); pReq->pid = htonl(1234); - strcpy(pReq->app, "test01"); + strcpy(pReq->app, "dndTestProfile"); strcpy(pReq->db, "invalid_db"); SRpcMsg rpcMsg = {0}; @@ -243,7 +243,7 @@ TEST_F(DndTestProfile, SHeartBeatMsg_01) { pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "test01"); + strcpy(pReq->app, "dndTestProfile"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; @@ -300,7 +300,7 @@ TEST_F(DndTestProfile, SKillConnMsg_01) { pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "test01"); + strcpy(pReq->app, "dndTestProfile"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; @@ -317,7 +317,7 @@ TEST_F(DndTestProfile, SKillConnMsg_01) { { SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); pReq->pid = htonl(1234); - strcpy(pReq->app, "test01"); + strcpy(pReq->app, "dndTestProfile"); strcpy(pReq->db, ""); SRpcMsg rpcMsg = {0}; @@ -394,7 +394,7 @@ TEST_F(DndTestProfile, SKillQueryMsg_01) { pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "test01"); + strcpy(pReq->app, "dndTestProfile"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; @@ -581,7 +581,7 @@ TEST_F(DndTestProfile, SKillStreamMsg_01) { pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "test01"); + strcpy(pReq->app, "dndTestProfile"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; diff --git a/source/dnode/mgmt/impl/test/show/CMakeLists.txt b/source/dnode/mgmt/impl/test/show/CMakeLists.txt new file mode 100644 index 0000000000..5f1596ce95 --- /dev/null +++ b/source/dnode/mgmt/impl/test/show/CMakeLists.txt @@ -0,0 +1,29 @@ +add_executable(dndTestShow "") + +target_sources(dndTestShow + PRIVATE + "show.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dndTestShow + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dndTestShow + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +enable_testing() + +add_test( + NAME dndTestShow + COMMAND dndTestShow +) diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp new file mode 100644 index 0000000000..1d04df8c00 --- /dev/null +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "deploy.h" + +class DndTestShow : public ::testing::Test { + protected: + void SetUp() override {} + void TearDown() override {} + + static void SetUpTestSuite() { + const char* user = "root"; + const char* pass = "taosdata"; + const char* path = "/tmp/dndTestShow"; + const char* fqdn = "localhost"; + uint16_t port = 9528; + + pServer = createServer(path, fqdn, port); + ASSERT(pServer); + pClient = createClient(user, pass, fqdn, port); + } + + static void TearDownTestSuite() { + dropServer(pServer); + dropClient(pClient); + } + + static SServer* pServer; + static SClient* pClient; + static int32_t connId; +}; + +SServer* DndTestShow::pServer; +SClient* DndTestShow::pClient; +int32_t DndTestShow::connId; + +TEST_F(DndTestShow, SShowMsg_01) { + ASSERT_NE(pClient, nullptr); + + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + pReq->pid = htonl(1234); + strcpy(pReq->app, "dndTestShow"); + strcpy(pReq->db, ""); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SConnectMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + + SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->connId = htonl(pRsp->connId); + + EXPECT_EQ(pRsp->connId, 1); + connId = pRsp->connId; +} + +TEST_F(DndTestShow, SShowMsg_02) { + ASSERT_NE(pClient, nullptr); + + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pReq->type = TSDB_MGMT_TABLE_MAX; + strcpy(pReq->db, ""); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SShowMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); +} + +TEST_F(DndTestShow, SShowMsg_03) { + ASSERT_NE(pClient, nullptr); + + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pReq->type = TSDB_MGMT_TABLE_START; + strcpy(pReq->db, ""); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SShowMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); +} + +TEST_F(DndTestShow, SShowMsg_04) { + ASSERT_NE(pClient, nullptr); + int32_t showId = 0; + + { + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pReq->type = TSDB_MGMT_TABLE_CONNS; + strcpy(pReq->db, ""); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SShowMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + + SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->showId = htonl(pRsp->showId); + STableMetaMsg* pMeta = &pRsp->tableMeta; + pMeta->contLen = htonl(pMeta->contLen); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tid = htonl(pMeta->tid); + pMeta->uid = htobe64(pMeta->uid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pRsp->showId; + + EXPECT_NE(pRsp->showId, 0); + EXPECT_EQ(pMeta->contLen, 0); + EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->numOfColumns, 7); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tid, 0); + EXPECT_EQ(pMeta->uid, 0); + EXPECT_STREQ(pMeta->sTableName, ""); + EXPECT_EQ(pMeta->suid, 0); + + SSchema* pSchema = NULL; + pSchema = &pMeta->schema[0]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); + EXPECT_EQ(pSchema->bytes, 4); + EXPECT_STREQ(pSchema->name, "connId"); + + pSchema = &pMeta->schema[1]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "user"); + + pSchema = &pMeta->schema[2]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "program"); + + pSchema = &pMeta->schema[3]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); + EXPECT_EQ(pSchema->bytes, 4); + EXPECT_STREQ(pSchema->name, "pid"); + + pSchema = &pMeta->schema[4]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "ip:port"); + + pSchema = &pMeta->schema[5]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); + EXPECT_EQ(pSchema->bytes, 8); + EXPECT_STREQ(pSchema->name, "login_time"); + + pSchema = &pMeta->schema[6]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); + EXPECT_EQ(pSchema->bytes, 8); + EXPECT_STREQ(pSchema->name, "last_access"); + } + + { + SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pReq->showId = htonl(showId); + pReq->free = 0; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SRetrieveTableMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->offset = htobe64(pRsp->offset); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->compLen = htonl(pRsp->compLen); + + EXPECT_EQ(pRsp->numOfRows, 1); + EXPECT_EQ(pRsp->offset, 0); + EXPECT_EQ(pRsp->useconds, 0); + EXPECT_EQ(pRsp->completed, 1); + EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRsp->compressed, 0); + EXPECT_EQ(pRsp->reserved, 0); + EXPECT_EQ(pRsp->compLen, 0); + } +} From c57e5d498508910a8de6151ddf546c3de2d03bcd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 6 Dec 2021 16:19:12 +0800 Subject: [PATCH 14/16] TD-10431 acct test --- include/common/taosmsg.h | 21 ++-- source/dnode/mgmt/impl/test/CMakeLists.txt | 3 +- .../dnode/mgmt/impl/test/acct/CMakeLists.txt | 29 +++++ source/dnode/mgmt/impl/test/acct/acct.cpp | 112 ++++++++++++++++++ 4 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/acct/CMakeLists.txt create mode 100644 source/dnode/mgmt/impl/test/acct/acct.cpp diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 300de7b698..ad68c0bfe1 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -360,10 +360,10 @@ typedef struct { } SConnectMsg; typedef struct SEpSet { - int8_t inUse; - int8_t numOfEps; - uint16_t port[TSDB_MAX_REPLICA]; - char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; + int8_t inUse; + int8_t numOfEps; + uint16_t port[TSDB_MAX_REPLICA]; + char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; } SEpSet; typedef struct { @@ -383,14 +383,9 @@ typedef struct { int32_t maxUsers; int32_t maxDbs; int32_t maxTimeSeries; - int32_t maxConnections; int32_t maxStreams; - int32_t maxPointsPerSecond; - int64_t maxStorage; // In unit of GB - int64_t maxQueryTime; // In unit of hour - int64_t maxInbound; - int64_t maxOutbound; - int8_t accessState; // Configured only by command + int64_t maxStorage; // In unit of GB + int32_t accessState; // Configured only by command } SCreateAcctMsg, SAlterAcctMsg; typedef struct { @@ -398,8 +393,8 @@ typedef struct { } SDropUserMsg, SDropAcctMsg; typedef struct { - char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char user[TSDB_USER_LEN]; + char pass[TSDB_KEY_LEN]; } SCreateUserMsg, SAlterUserMsg; typedef struct { diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 84edabfc58..836de04735 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,2 +1,3 @@ +add_subdirectory(acct) add_subdirectory(profile) -add_subdirectory(show) \ No newline at end of file +add_subdirectory(show) diff --git a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt new file mode 100644 index 0000000000..3e963df2e6 --- /dev/null +++ b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt @@ -0,0 +1,29 @@ +add_executable(dndTestAcct "") + +target_sources(dndTestAcct + PRIVATE + "acct.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dndTestAcct + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dndTestAcct + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +enable_testing() + +add_test( + NAME dndTestAcct + COMMAND dndTestAcct +) diff --git a/source/dnode/mgmt/impl/test/acct/acct.cpp b/source/dnode/mgmt/impl/test/acct/acct.cpp new file mode 100644 index 0000000000..cda44a3be8 --- /dev/null +++ b/source/dnode/mgmt/impl/test/acct/acct.cpp @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "deploy.h" + +class DndTestAcct : public ::testing::Test { + protected: + void SetUp() override {} + void TearDown() override {} + + static void SetUpTestSuite() { + const char* user = "root"; + const char* pass = "taosdata"; + const char* path = "/tmp/dndTestAcct"; + const char* fqdn = "localhost"; + uint16_t port = 9520; + + pServer = createServer(path, fqdn, port); + ASSERT(pServer); + pClient = createClient(user, pass, fqdn, port); + } + + static void TearDownTestSuite() { + dropServer(pServer); + dropClient(pClient); + } + + static SServer* pServer; + static SClient* pClient; + static int32_t connId; +}; + +SServer* DndTestAcct::pServer; +SClient* DndTestAcct::pClient; +int32_t DndTestAcct::connId; + +TEST_F(DndTestAcct, CreateAcct) { + ASSERT_NE(pClient, nullptr); + + SCreateAcctMsg* pReq = (SCreateAcctMsg*)rpcMallocCont(sizeof(SCreateAcctMsg)); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateAcctMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_ACCT; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); +} + +TEST_F(DndTestAcct, AlterAcct) { + ASSERT_NE(pClient, nullptr); + + SAlterAcctMsg* pReq = (SAlterAcctMsg*)rpcMallocCont(sizeof(SAlterAcctMsg)); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterAcctMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_ACCT; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); +} + +TEST_F(DndTestAcct, DropAcct) { + ASSERT_NE(pClient, nullptr); + + SDropAcctMsg* pReq = (SDropAcctMsg*)rpcMallocCont(sizeof(SDropAcctMsg)); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropAcctMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_ACCT; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); +} + +TEST_F(DndTestAcct, ShowAcct) { + ASSERT_NE(pClient, nullptr); + + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pReq->type = TSDB_MGMT_TABLE_ACCT; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SShowMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); +} \ No newline at end of file From b5e9ba4a842f576160bed2f97e24516347454ab5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 6 Dec 2021 18:21:14 +0800 Subject: [PATCH 15/16] TD-10431 cluster test --- source/dnode/mgmt/impl/test/CMakeLists.txt | 1 + .../mgmt/impl/test/cluster/CMakeLists.txt | 29 +++ .../dnode/mgmt/impl/test/cluster/cluster.cpp | 170 ++++++++++++++++ .../dnode/mgmt/impl/test/profile/profile.cpp | 12 +- source/dnode/mgmt/impl/test/show/show.cpp | 2 +- source/dnode/mgmt/impl/test/sut/deploy.cpp | 4 +- source/dnode/mgmt/impl/test/sut/deploy.h | 1 + source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndCluster.c | 184 ++++++++++-------- 9 files changed, 314 insertions(+), 91 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/cluster/CMakeLists.txt create mode 100644 source/dnode/mgmt/impl/test/cluster/cluster.cpp diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 836de04735..e7aa103996 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,3 +1,4 @@ add_subdirectory(acct) +add_subdirectory(cluster) add_subdirectory(profile) add_subdirectory(show) diff --git a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt new file mode 100644 index 0000000000..b513f290e4 --- /dev/null +++ b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt @@ -0,0 +1,29 @@ +add_executable(dndTestCluster "") + +target_sources(dndTestCluster + PRIVATE + "cluster.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dndTestCluster + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dndTestCluster + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +enable_testing() + +add_test( + NAME dndTestCluster + COMMAND dndTestCluster +) diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp new file mode 100644 index 0000000000..d7f9bca5ed --- /dev/null +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "deploy.h" + +class DndTestCluster : public ::testing::Test { + protected: + void SetUp() override {} + void TearDown() override {} + + static void SetUpTestSuite() { + const char* user = "root"; + const char* pass = "taosdata"; + const char* path = "/tmp/dndTestCluster"; + const char* fqdn = "localhost"; + uint16_t port = 9521; + + pServer = createServer(path, fqdn, port); + ASSERT(pServer); + pClient = createClient(user, pass, fqdn, port); + } + + static void TearDownTestSuite() { + dropServer(pServer); + dropClient(pClient); + } + + static SServer* pServer; + static SClient* pClient; + static int32_t connId; +}; + +SServer* DndTestCluster::pServer; +SClient* DndTestCluster::pClient; +int32_t DndTestCluster::connId; + +TEST_F(DndTestCluster, ShowCluster) { + ASSERT_NE(pClient, nullptr); + int32_t showId = 0; + + { + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pReq->type = TSDB_MGMT_TABLE_CLUSTER; + strcpy(pReq->db, ""); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SShowMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + + SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->showId = htonl(pRsp->showId); + STableMetaMsg* pMeta = &pRsp->tableMeta; + pMeta->contLen = htonl(pMeta->contLen); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tid = htonl(pMeta->tid); + pMeta->uid = htobe64(pMeta->uid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pRsp->showId; + + EXPECT_NE(pRsp->showId, 0); + EXPECT_EQ(pMeta->contLen, 0); + EXPECT_STREQ(pMeta->tableFname, "show cluster"); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->numOfColumns, 3); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tid, 0); + EXPECT_EQ(pMeta->uid, 0); + EXPECT_STREQ(pMeta->sTableName, ""); + EXPECT_EQ(pMeta->suid, 0); + + SSchema* pSchema = NULL; + pSchema = &pMeta->schema[0]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); + EXPECT_EQ(pSchema->bytes, 4); + EXPECT_STREQ(pSchema->name, "id"); + + pSchema = &pMeta->schema[1]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "name"); + + pSchema = &pMeta->schema[2]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); + EXPECT_EQ(pSchema->bytes, 8); + EXPECT_STREQ(pSchema->name, "create_time"); + } + + { + SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pReq->showId = htonl(showId); + pReq->free = 0; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SRetrieveTableMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; + ASSERT_NE(pRsp, nullptr); + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->offset = htobe64(pRsp->offset); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->compLen = htonl(pRsp->compLen); + + EXPECT_EQ(pRsp->numOfRows, 1); + EXPECT_EQ(pRsp->offset, 0); + EXPECT_EQ(pRsp->useconds, 0); + EXPECT_EQ(pRsp->completed, 1); + EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRsp->compressed, 0); + EXPECT_EQ(pRsp->reserved, 0); + EXPECT_EQ(pRsp->compLen, 0); + + char* pData = pRsp->data; + int32_t pos = 0; + + int32_t id = *((int32_t*)(pData + pos)); + pos += sizeof(int32_t); + + int32_t nameLen = varDataLen(pData + pos); + pos += sizeof(VarDataLenT); + + char* name = (char*)(pData + pos); + pos += TSDB_CLUSTER_ID_LEN; + + int64_t create_time = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + + EXPECT_NE(id, 0); + EXPECT_EQ(nameLen, 36); + EXPECT_STRNE(name, ""); + EXPECT_GT(create_time, 0); + printf("--- id:%d nameLen:%d name:%s time:%" PRId64 " --- \n", id, nameLen, name, create_time); + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 7c0e79cd69..14e8dda457 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -25,7 +25,7 @@ class DndTestProfile : public ::testing::Test { const char* pass = "taosdata"; const char* path = "/tmp/dndTestProfile"; const char* fqdn = "localhost"; - uint16_t port = 9527; + uint16_t port = 9522; pServer = createServer(path, fqdn, port); ASSERT(pServer); @@ -79,7 +79,7 @@ TEST_F(DndTestProfile, SConnectMsg_01) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_EQ(pRsp->epSet.port[0], 9522); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; @@ -272,7 +272,7 @@ TEST_F(DndTestProfile, SHeartBeatMsg_01) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_EQ(pRsp->epSet.port[0], 9522); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } @@ -344,7 +344,7 @@ TEST_F(DndTestProfile, SKillConnMsg_01) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_EQ(pRsp->epSet.port[0], 9522); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; @@ -423,7 +423,7 @@ TEST_F(DndTestProfile, SKillQueryMsg_01) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_EQ(pRsp->epSet.port[0], 9522); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } } @@ -610,7 +610,7 @@ TEST_F(DndTestProfile, SKillStreamMsg_01) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_EQ(pRsp->epSet.port[0], 9522); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } } diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index 1d04df8c00..e8c8fa261c 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -25,7 +25,7 @@ class DndTestShow : public ::testing::Test { const char* pass = "taosdata"; const char* path = "/tmp/dndTestShow"; const char* fqdn = "localhost"; - uint16_t port = 9528; + uint16_t port = 9523; pServer = createServer(path, fqdn, port); ASSERT(pServer); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 6a7732b7ed..ad1667eac5 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -113,14 +113,14 @@ SClient* createClient(const char* user, const char* pass, const char* fqdn, uint SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "DND-C"; + rpcInit.label = (char*)"DND-C"; rpcInit.numOfThreads = 1; rpcInit.cfp = processClientRsp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = 30 * 1000; rpcInit.user = (char*)user; - rpcInit.ckey = "key"; + rpcInit.ckey = (char*)"key"; rpcInit.parent = pClient; rpcInit.secret = (char*)secretEncrypt; rpcInit.parent = pClient; diff --git a/source/dnode/mgmt/impl/test/sut/deploy.h b/source/dnode/mgmt/impl/test/sut/deploy.h index 8a352b96e8..93188458ec 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.h +++ b/source/dnode/mgmt/impl/test/sut/deploy.h @@ -24,6 +24,7 @@ #include "trpc.h" #include "tthread.h" #include "ulog.h" +#include "tdataformat.h" typedef struct { SDnode* pDnode; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 6ad0f05c2a..9e4f922540 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -116,7 +116,7 @@ typedef struct STrans { typedef struct SClusterObj { int32_t id; - char uid[TSDB_CLUSTER_ID_LEN]; + char name[TSDB_CLUSTER_ID_LEN]; int64_t createdTime; int64_t updateTime; } SClusterObj; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 6013ac0c31..c9ef7ce87d 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -15,11 +15,39 @@ #define _DEFAULT_SOURCE #include "mndCluster.h" -#include "mndTrans.h" #include "mndShow.h" +#include "mndTrans.h" #define SDB_CLUSTER_VER 1 +static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); +static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw); +static int32_t mndClusterActionInsert(SSdb *pSdb, SClusterObj *pCluster); +static int32_t mndClusterActionDelete(SSdb *pSdb, SClusterObj *pCluster); +static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pSrcCluster, SClusterObj *pDstCluster); +static int32_t mndCreateDefaultCluster(SMnode *pMnode); +static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter); + +int32_t mndInitCluster(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_CLUSTER, + .keyType = SDB_KEY_INT32, + .deployFp = (SdbDeployFp)mndCreateDefaultCluster, + .encodeFp = (SdbEncodeFp)mndClusterActionEncode, + .decodeFp = (SdbDecodeFp)mndClusterActionDecode, + .insertFp = (SdbInsertFp)mndClusterActionInsert, + .updateFp = (SdbUpdateFp)mndClusterActionUpdate, + .deleteFp = (SdbDeleteFp)mndClusterActionDelete}; + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_CLUSTER, mndGetClusterMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CLUSTER, mndRetrieveClusters); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CLUSTER, mndCancelGetNextCluster); + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupCluster(SMnode *pMnode) {} + static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, SDB_CLUSTER_VER, sizeof(SClusterObj)); if (pRaw == NULL) return NULL; @@ -28,7 +56,7 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { SDB_SET_INT32(pRaw, dataPos, pCluster->id); SDB_SET_INT64(pRaw, dataPos, pCluster->createdTime) SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime) - SDB_SET_BINARY(pRaw, dataPos, pCluster->uid, TSDB_CLUSTER_ID_LEN) + SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN) return pRaw; } @@ -51,7 +79,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, pRow, dataPos, &pCluster->id) SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->updateTime) - SDB_GET_BINARY(pRaw, pRow, dataPos, pCluster->uid, TSDB_CLUSTER_ID_LEN) + SDB_GET_BINARY(pRaw, pRow, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN) return pRow; } @@ -76,14 +104,14 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { clusterObj.createdTime = taosGetTimestampMs(); clusterObj.updateTime = clusterObj.createdTime; - int32_t code = taosGetSystemUid(clusterObj.uid, TSDB_CLUSTER_ID_LEN); + int32_t code = taosGetSystemUid(clusterObj.name, TSDB_CLUSTER_ID_LEN); if (code != 0) { - strcpy(clusterObj.uid, "tdengine2.0"); - mError("failed to get uid from system, set to default val %s", clusterObj.uid); + strcpy(clusterObj.name, "tdengine2.0"); + mError("failed to get name from system, set to default val %s", clusterObj.name); } else { - mDebug("cluster:%d, uid is %s", clusterObj.id, clusterObj.uid); + mDebug("cluster:%d, name is %s", clusterObj.id, clusterObj.name); } - clusterObj.id = MurmurHash3_32(clusterObj.uid, TSDB_CLUSTER_ID_LEN); + clusterObj.id = MurmurHash3_32(clusterObj.name, TSDB_CLUSTER_ID_LEN); clusterObj.id = abs(clusterObj.id); pMnode->clusterId = clusterObj.id; @@ -95,85 +123,79 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { return sdbWrite(pMnode->pSdb, pRaw); } +static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; -// static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { -// int32_t cols = 0; -// SSchema *pSchema = pMeta->schema; + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; -// pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE; -// pSchema[cols].type = TSDB_DATA_TYPE_BINARY; -// strcpy(pSchema[cols].name, "clusterId"); -// pSchema[cols].bytes = htons(pShow->bytes[cols]); -// cols++; + pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; -// pShow->bytes[cols] = 8; -// pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; -// strcpy(pSchema[cols].name, "create_time"); -// pSchema[cols].bytes = htons(pShow->bytes[cols]); -// cols++; + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; -// pMeta->numOfColumns = htons(cols); -// strcpy(pMeta->tableFname, "show cluster"); -// pShow->numOfColumns = cols; + pMeta->numOfColumns = htons(cols); + strcpy(pMeta->tableFname, "show cluster"); + pShow->numOfColumns = cols; -// pShow->offset[0] = 0; -// for (int32_t i = 1; i < cols; ++i) { -// pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; -// } + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } -// pShow->numOfRows = 1; -// pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + pShow->numOfRows = 1; + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; -// return 0; -// } - -// static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn) { -// int32_t numOfRows = 0; -// int32_t cols = 0; -// char * pWrite; -// SClusterObj *pCluster = NULL; - -// while (numOfRows < rows) { -// pShow->pIter = mnodeGetNextCluster(pShow->pIter, &pCluster); -// if (pCluster == NULL) break; - -// cols = 0; - -// pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; -// STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pCluster->uid, TSDB_CLUSTER_ID_LEN); -// cols++; - -// pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; -// *(int64_t *) pWrite = pCluster->createdTime; -// cols++; - -// mnodeDecClusterRef(pCluster); -// numOfRows++; -// } - -// mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); -// pShow->numOfReads += numOfRows; -// return numOfRows; -// } - -// static void mnodeCancelGetNextCluster(void *pIter) { -// sdbFreeIter(tsClusterSdb, pIter); -// } - -int32_t mndInitCluster(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_CLUSTER, - .keyType = SDB_KEY_INT32, - .deployFp = (SdbDeployFp)mndCreateDefaultCluster, - .encodeFp = (SdbEncodeFp)mndClusterActionEncode, - .decodeFp = (SdbDecodeFp)mndClusterActionDecode, - .insertFp = (SdbInsertFp)mndClusterActionInsert, - .updateFp = (SdbUpdateFp)mndClusterActionUpdate, - .deleteFp = (SdbDeleteFp)mndClusterActionDelete}; - - // mndAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta); - // mndAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters); - // mndAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster); - return sdbSetTable(pMnode->pSdb, table); + return 0; } -void mndCleanupCluster(SMnode *pMnode) {} +static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + char *pWrite; + SClusterObj *pCluster = NULL; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_CLUSTER, pShow->pIter, (void **)&pCluster); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pCluster->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pCluster->name, TSDB_CLUSTER_ID_LEN); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pCluster->createdTime; + cols++; + + sdbRelease(pSdb, pCluster); + numOfRows++; + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file From d132653f61683bb7bf5537ec0d59dc9dd53239a6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 6 Dec 2021 19:57:13 +0800 Subject: [PATCH 16/16] TD-10431 telemetry coding --- include/dnode/mgmt/dnode.h | 3 + include/dnode/mnode/mnode.h | 21 ++-- include/util/taoserror.h | 4 + include/util/tversion.h | 31 ++++++ source/common/src/versionUtil.c | 63 ----------- source/dnode/mgmt/daemon/src/daemon.c | 4 +- source/dnode/mgmt/impl/src/dndMnode.c | 17 +-- source/dnode/mnode/impl/inc/mndCluster.h | 1 + source/dnode/mnode/impl/inc/mndInt.h | 18 ++-- source/dnode/mnode/impl/src/mndCluster.c | 15 ++- source/dnode/mnode/impl/src/mndDnode.c | 25 ++--- source/dnode/mnode/impl/src/mndProfile.c | 6 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndTelem.c | 128 ++++++++++++----------- source/dnode/mnode/impl/src/mnode.c | 29 ++--- source/util/src/terror.c | 3 + source/util/src/tversion.c | 93 ++++++++++++++++ 17 files changed, 286 insertions(+), 177 deletions(-) create mode 100644 include/util/tversion.h delete mode 100644 source/common/src/versionUtil.c create mode 100644 source/util/src/tversion.c diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 50886932ce..540a501f3a 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -31,6 +31,7 @@ typedef struct { int16_t numOfSupportMnodes; int16_t numOfSupportVnodes; int16_t numOfSupportQnodes; + int8_t enableTelem; int32_t statusInterval; int32_t mnodeEqualVnodeNum; float numOfThreadsPerCore; @@ -45,6 +46,8 @@ typedef struct { char timezone[TSDB_TIMEZONE_LEN]; char locale[TSDB_LOCALE_LEN]; char charset[TSDB_LOCALE_LEN]; + char buildinfo[64]; + char gitinfo[48]; } SDnodeOpt; /* ------------------------ SDnode ------------------------ */ diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 0a897b93f8..8a8346fbc0 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -43,24 +43,31 @@ typedef struct SMnodeLoad { int64_t compStorage; } SMnodeLoad; +typedef struct SMnodeCfg { + int32_t sver; + int8_t enableTelem; + int32_t statusInterval; + int32_t mnodeEqualVnodeNum; + int32_t shellActivityTimer; + char *timezone; + char *locale; + char *charset; + char *buildinfo; + char *gitinfo; +} SMnodeCfg; + typedef struct { int32_t dnodeId; int32_t clusterId; int8_t replica; int8_t selfIndex; SReplica replicas[TSDB_MAX_REPLICA]; + SMnodeCfg cfg; SDnode *pDnode; PutMsgToMnodeQFp putMsgToApplyMsgFp; SendMsgToDnodeFp sendMsgToDnodeFp; SendMsgToMnodeFp sendMsgToMnodeFp; SendRedirectMsgFp sendRedirectMsgFp; - int32_t sver; - int32_t statusInterval; - int32_t mnodeEqualVnodeNum; - int32_t shellActivityTimer; - char *timezone; - char *locale; - char *charset; } SMnodeOpt; /* ------------------------ SMnode ------------------------ */ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 413c84fe31..f6cefa96df 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -76,6 +76,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0114) #define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0115) +#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x0120) +#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x0121) +#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x0122) + //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) //"Invalid Operation") #define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201) //"Invalid qhandle") diff --git a/include/util/tversion.h b/include/util/tversion.h new file mode 100644 index 0000000000..3d7a7e1b66 --- /dev/null +++ b/include/util/tversion.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_VERSION_H +#define _TD_UTIL_VERSION_H + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t taosVersionStrToInt(const char *vstr, int32_t *vint); +int32_t taosVersionIntToStr(int32_t vint, char *vstr, int32_t len); +int32_t taosCheckVersionCompatible(int32_t clientVer, int32_t serverVer, int32_t comparedSegments); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_VERSION_H*/ diff --git a/source/common/src/versionUtil.c b/source/common/src/versionUtil.c deleted file mode 100644 index 22c50fa5cf..0000000000 --- a/source/common/src/versionUtil.c +++ /dev/null @@ -1,63 +0,0 @@ -#include "os.h" -#include "tdef.h" -#include "ulog.h" -#include "taoserror.h" - -bool taosGetVersionNumber(char *versionStr, int *versionNubmer) { - if (versionStr == NULL || versionNubmer == NULL) { - return false; - } - - int versionNumberPos[5] = {0}; - int len = (int)strlen(versionStr); - int dot = 0; - for (int pos = 0; pos < len && dot < 4; ++pos) { - if (versionStr[pos] == '.') { - versionStr[pos] = 0; - versionNumberPos[++dot] = pos + 1; - } - } - - if (dot != 3) { - return false; - } - - for (int pos = 0; pos < 4; ++pos) { - versionNubmer[pos] = atoi(versionStr + versionNumberPos[pos]); - } - versionStr[versionNumberPos[1] - 1] = '.'; - versionStr[versionNumberPos[2] - 1] = '.'; - versionStr[versionNumberPos[3] - 1] = '.'; - - return true; -} - -int taosCheckVersion(char *input_client_version, char *input_server_version, int comparedSegments) { - char client_version[TSDB_VERSION_LEN] = {0}; - char server_version[TSDB_VERSION_LEN] = {0}; - int clientVersionNumber[4] = {0}; - int serverVersionNumber[4] = {0}; - - tstrncpy(client_version, input_client_version, sizeof(client_version)); - tstrncpy(server_version, input_server_version, sizeof(server_version)); - - if (!taosGetVersionNumber(client_version, clientVersionNumber)) { - uError("invalid client version:%s", client_version); - return TSDB_CODE_TSC_INVALID_VERSION; - } - - if (!taosGetVersionNumber(server_version, serverVersionNumber)) { - uError("invalid server version:%s", server_version); - return TSDB_CODE_TSC_INVALID_VERSION; - } - - for(int32_t i = 0; i < comparedSegments; ++i) { - if (clientVersionNumber[i] != serverVersionNumber[i]) { - uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version, - client_version); - return TSDB_CODE_TSC_INVALID_VERSION; - } - } - - return 0; -} diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index 08624f20f4..cf9a960062 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -136,7 +136,7 @@ void dmnWaitSignal() { } void dmnInitOption(SDnodeOpt *pOption) { - pOption->sver = tsVersion; + pOption->sver = 30000000; //3.0.0.0 pOption->numOfCores = tsNumOfCores; pOption->numOfSupportMnodes = 1; pOption->numOfSupportVnodes = 1; @@ -155,6 +155,8 @@ void dmnInitOption(SDnodeOpt *pOption) { tstrncpy(pOption->timezone, tsTimezone, TSDB_TIMEZONE_LEN); tstrncpy(pOption->locale, tsLocale, TSDB_LOCALE_LEN); tstrncpy(pOption->charset, tsCharset, TSDB_LOCALE_LEN); + tstrncpy(pOption->buildinfo, buildinfo, 64); + tstrncpy(pOption->gitinfo, gitinfo, 48); } int dmnRunDnode() { diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 4afce4655d..374a2f2b2c 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -331,13 +331,16 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->putMsgToApplyMsgFp = dndPutMsgIntoMnodeApplyQueue; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = pDnode->opt.sver; - pOption->statusInterval = pDnode->opt.statusInterval; - pOption->mnodeEqualVnodeNum = pDnode->opt.mnodeEqualVnodeNum; - pOption->shellActivityTimer = pDnode->opt.shellActivityTimer; - pOption->timezone = pDnode->opt.timezone; - pOption->charset = pDnode->opt.charset; - pOption->locale = pDnode->opt.locale; + pOption->cfg.sver = pDnode->opt.sver; + pOption->cfg.enableTelem = pDnode->opt.enableTelem; + pOption->cfg.statusInterval = pDnode->opt.statusInterval; + pOption->cfg.mnodeEqualVnodeNum = pDnode->opt.mnodeEqualVnodeNum; + pOption->cfg.shellActivityTimer = pDnode->opt.shellActivityTimer; + pOption->cfg.timezone = pDnode->opt.timezone; + pOption->cfg.charset = pDnode->opt.charset; + pOption->cfg.locale = pDnode->opt.locale; + pOption->cfg.gitinfo = pDnode->opt.gitinfo; + pOption->cfg.buildinfo = pDnode->opt.buildinfo; } static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index efc1ada67d..0206695b88 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -24,6 +24,7 @@ extern "C" { int32_t mndInitCluster(SMnode *pMnode); void mndCleanupCluster(SMnode *pMnode); +int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index dbca6ad057..8d20356e39 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -51,6 +51,15 @@ typedef struct { SCacheObj *cache; } SProfileMgmt; +typedef struct { + int8_t enable; + pthread_mutex_t lock; + pthread_cond_t cond; + volatile int32_t exit; + pthread_t thread; + char email[TSDB_FQDN_LEN]; +} STelemMgmt; + typedef struct SMnode { int32_t dnodeId; int32_t clusterId; @@ -59,23 +68,18 @@ typedef struct SMnode { SReplica replicas[TSDB_MAX_REPLICA]; tmr_h timer; char *path; + SMnodeCfg cfg; SSdb *pSdb; SDnode *pDnode; SArray *pSteps; SShowMgmt showMgmt; SProfileMgmt profileMgmt; + STelemMgmt telemMgmt; MndMsgFp msgFp[TSDB_MSG_TYPE_MAX]; SendMsgToDnodeFp sendMsgToDnodeFp; SendMsgToMnodeFp sendMsgToMnodeFp; SendRedirectMsgFp sendRedirectMsgFp; PutMsgToMnodeQFp putMsgToApplyMsgFp; - int32_t sver; - int32_t statusInterval; - int32_t mnodeEqualVnodeNum; - int32_t shellActivityTimer; - char *timezone; - char *locale; - char *charset; } SMnode; void mndSendMsgToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *rpcMsg); diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index c9ef7ce87d..9fa7a06630 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -48,6 +48,19 @@ int32_t mndInitCluster(SMnode *pMnode) { void mndCleanupCluster(SMnode *pMnode) {} +int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { + SSdb *pSdb = pMnode->pSdb; + + SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId); + if (pCluster = NULL) { + return -1; + } + + tstrncpy(clusterName, pCluster->name, len); + sdbRelease(pSdb, pCluster); + return 0; +} + static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, SDB_CLUSTER_VER, sizeof(SClusterObj)); if (pRaw == NULL) return NULL; @@ -198,4 +211,4 @@ static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 24e6113161..241b006293 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -179,32 +179,33 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) { } static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { - if (pCfg->mnodeEqualVnodeNum != pMnode->mnodeEqualVnodeNum) { - mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg inconsistent", pCfg->mnodeEqualVnodeNum, pMnode->mnodeEqualVnodeNum); + if (pCfg->mnodeEqualVnodeNum != pMnode->cfg.mnodeEqualVnodeNum) { + mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg inconsistent", pCfg->mnodeEqualVnodeNum, + pMnode->cfg.mnodeEqualVnodeNum); return DND_REASON_MN_EQUAL_VN_NOT_MATCH; } - if (pCfg->statusInterval != pMnode->statusInterval) { - mError("\"statusInterval\"[%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->statusInterval); + if (pCfg->statusInterval != pMnode->cfg.statusInterval) { + mError("\"statusInterval\"[%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->cfg.statusInterval); return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } int64_t checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - if ((0 != strcasecmp(pCfg->timezone, pMnode->timezone)) && (checkTime != pCfg->checkTime)) { - mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, tsTimezone, + if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (checkTime != pCfg->checkTime)) { + mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, pCfg->checkTime, checkTime); return DND_REASON_TIME_ZONE_NOT_MATCH; } - if (0 != strcasecmp(pCfg->locale, pMnode->locale)) { - mError("\"locale\"[%s - %s] cfg parameters inconsistent", pCfg->locale, pMnode->locale); + if (0 != strcasecmp(pCfg->locale, pMnode->cfg.locale)) { + mError("\"locale\"[%s - %s] cfg parameters inconsistent", pCfg->locale, pMnode->cfg.locale); return DND_REASON_LOCALE_NOT_MATCH; } - if (0 != strcasecmp(pCfg->charset, pMnode->charset)) { - mError("\"charset\"[%s - %s] cfg parameters inconsistent.", pCfg->charset, pMnode->charset); + if (0 != strcasecmp(pCfg->charset, pMnode->cfg.charset)) { + mError("\"charset\"[%s - %s] cfg parameters inconsistent.", pCfg->charset, pMnode->cfg.charset); return DND_REASON_CHARSET_NOT_MATCH; } @@ -251,12 +252,12 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } } - if (pStatus->sver != pMnode->sver) { + if (pStatus->sver != pMnode->cfg.sver) { if (pDnode != NULL && pDnode->status != DND_STATUS_READY) { pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; } mndReleaseDnode(pMnode, pDnode); - mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->sver); + mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->cfg.sver); return TSDB_CODE_MND_INVALID_MSG_VERSION; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 8f51b1a0be..ad7a6322bb 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -67,7 +67,7 @@ static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - int32_t connCheckTime = pMnode->shellActivityTimer * 2; + int32_t connCheckTime = pMnode->cfg.shellActivityTimer * 2; pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->cache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -126,7 +126,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t tstrncpy(connObj.user, user, TSDB_USER_LEN); tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); - int32_t keepTime = pMnode->shellActivityTimer * 3; + int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); if (pConn == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -153,7 +153,7 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) { return NULL; } - int32_t keepTime = pMnode->shellActivityTimer * 3; + int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; pConn->lastAccess = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); mTrace("conn:%d, data:%p acquired from cache", pConn->id, pConn); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index ca0d9b9d50..747ea39237 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -69,7 +69,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg) { return NULL; } - int32_t keepTime = pMnode->shellActivityTimer * 6 * 1000; + int32_t keepTime = pMnode->cfg.shellActivityTimer * 6 * 1000; SShowObj *pShowRet = taosCachePut(pMgmt->cache, &showId, sizeof(int32_t), pShow, size, keepTime); free(pShow); if (pShowRet == NULL) { diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index f9f349aad8..663ae76506 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -15,27 +15,15 @@ #define _DEFAULT_SOURCE #include "mndTelem.h" -#include "tbuffer.h" -#include "tglobal.h" +#include "mndCluster.h" #include "mndSync.h" +#include "tbuffer.h" +#include "tversion.h" #define TELEMETRY_SERVER "telemetry.taosdata.com" #define TELEMETRY_PORT 80 #define REPORT_INTERVAL 86400 -/* - * sem_timedwait is NOT implemented on MacOSX - * thus we use pthread_mutex_t/pthread_cond_t to simulate - */ -static struct { - bool enable; - pthread_mutex_t lock; - pthread_cond_t cond; - volatile int32_t exit; - pthread_t thread; - char email[TSDB_FQDN_LEN]; -} tsTelem; - static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); } static void mndCloseObject(SBufferWriter* bw) { @@ -86,7 +74,7 @@ static void mndAddStringField(SBufferWriter* bw, const char* k, const char* v) { tbufWriteChar(bw, ','); } -static void mndAddCpuInfo(SBufferWriter* bw) { +static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; int32_t done = 0; @@ -116,7 +104,7 @@ static void mndAddCpuInfo(SBufferWriter* bw) { fclose(fp); } -static void mndAddOsInfo(SBufferWriter* bw) { +static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; @@ -142,7 +130,7 @@ static void mndAddOsInfo(SBufferWriter* bw) { fclose(fp); } -static void mndAddMemoryInfo(SBufferWriter* bw) { +static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; @@ -165,16 +153,21 @@ static void mndAddMemoryInfo(SBufferWriter* bw) { fclose(fp); } -static void mndAddVersionInfo(SBufferWriter* bw) { - mndAddStringField(bw, "version", version); - mndAddStringField(bw, "buildInfo", buildinfo); - mndAddStringField(bw, "gitInfo", gitinfo); - mndAddStringField(bw, "email", tsTelem.email); +static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) { + STelemMgmt* pMgmt = &pMnode->telemMgmt; + + char vstr[32] = {0}; + taosVersionIntToStr(pMnode->cfg.sver, vstr, 32); + + mndAddStringField(bw, "version", vstr); + mndAddStringField(bw, "buildInfo", pMnode->cfg.buildinfo); + mndAddStringField(bw, "gitInfo", pMnode->cfg.gitinfo); + mndAddStringField(bw, "email", pMgmt->email); } -static void mndAddRuntimeInfo(SBufferWriter* bw) { +static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) { SMnodeLoad load = {0}; - if (mndGetLoad(NULL, &load) != 0) { + if (mndGetLoad(pMnode, &load) != 0) { return; } @@ -190,11 +183,13 @@ static void mndAddRuntimeInfo(SBufferWriter* bw) { mndAddIntField(bw, "compStorage", load.compStorage); } -static void mndSendTelemetryReport() { +static void mndSendTelemetryReport(SMnode* pMnode) { + STelemMgmt* pMgmt = &pMnode->telemMgmt; + char buf[128] = {0}; uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); if (ip == 0xffffffff) { - mTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); + mTrace("failed to get IP address of " TELEMETRY_SERVER " since :%s", strerror(errno)); return; } SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); @@ -203,19 +198,18 @@ static void mndSendTelemetryReport() { return; } - int32_t clusterId = 0; - char clusterIdStr[20] = {0}; - snprintf(clusterIdStr, sizeof(clusterIdStr), "%d", clusterId); + char clusterName[64] = {0}; + mndGetClusterName(pMnode, clusterName, sizeof(clusterName)); SBufferWriter bw = tbufInitWriter(NULL, false); mndBeginObject(&bw); - mndAddStringField(&bw, "instanceId", clusterIdStr); + mndAddStringField(&bw, "instanceId", clusterName); mndAddIntField(&bw, "reportVersion", 1); - mndAddOsInfo(&bw); - mndAddCpuInfo(&bw); - mndAddMemoryInfo(&bw); - mndAddVersionInfo(&bw); - mndAddRuntimeInfo(&bw); + mndAddOsInfo(pMnode, &bw); + mndAddCpuInfo(pMnode, &bw); + mndAddMemoryInfo(pMnode, &bw); + mndAddVersionInfo(pMnode, &bw); + mndAddRuntimeInfo(pMnode, &bw); mndCloseObject(&bw); const char* header = @@ -241,23 +235,26 @@ static void mndSendTelemetryReport() { } static void* mndTelemThreadFp(void* param) { + SMnode* pMnode = param; + STelemMgmt* pMgmt = &pMnode->telemMgmt; + struct timespec end = {0}; clock_gettime(CLOCK_REALTIME, &end); end.tv_sec += 300; // wait 5 minutes before send first report setThreadName("mnd-telem"); - while (!tsTelem.exit) { + while (!pMgmt->exit) { int32_t r = 0; struct timespec ts = end; - pthread_mutex_lock(&tsTelem.lock); - r = pthread_cond_timedwait(&tsTelem.cond, &tsTelem.lock, &ts); - pthread_mutex_unlock(&tsTelem.lock); + pthread_mutex_lock(&pMgmt->lock); + r = pthread_cond_timedwait(&pMgmt->cond, &pMgmt->lock, &ts); + pthread_mutex_unlock(&pMgmt->lock); if (r == 0) break; if (r != ETIMEDOUT) continue; - if (mndIsMaster(NULL)) { - mndSendTelemetryReport(); + if (mndIsMaster(pMnode)) { + mndSendTelemetryReport(pMnode); } end.tv_sec += REPORT_INTERVAL; } @@ -265,35 +262,39 @@ static void* mndTelemThreadFp(void* param) { return NULL; } -static void mndGetEmail(char* filepath) { +static void mndGetEmail(SMnode* pMnode, char* filepath) { + STelemMgmt* pMgmt = &pMnode->telemMgmt; + int32_t fd = taosOpenFileRead(filepath); if (fd < 0) { return; } - if (taosReadFile(fd, (void*)tsTelem.email, TSDB_FQDN_LEN) < 0) { + if (taosReadFile(fd, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } taosCloseFile(fd); } -int32_t mndInitTelem(SMnode *pMnode) { - tsTelem.enable = tsEnableTelemetryReporting; - if (!tsTelem.enable) return 0; +int32_t mndInitTelem(SMnode* pMnode) { + STelemMgmt* pMgmt = &pMnode->telemMgmt; + pMgmt->enable = pMnode->cfg.enableTelem; - tsTelem.exit = 0; - pthread_mutex_init(&tsTelem.lock, NULL); - pthread_cond_init(&tsTelem.cond, NULL); - tsTelem.email[0] = 0; + if (!pMgmt->enable) return 0; - mndGetEmail("/usr/local/taos/email"); + pMgmt->exit = 0; + pthread_mutex_init(&pMgmt->lock, NULL); + pthread_cond_init(&pMgmt->cond, NULL); + pMgmt->email[0] = 0; + + mndGetEmail(pMnode, "/usr/local/taos/email"); pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - int32_t code = pthread_create(&tsTelem.thread, &attr, mndTelemThreadFp, NULL); + int32_t code = pthread_create(&pMgmt->thread, &attr, mndTelemThreadFp, pMnode); pthread_attr_destroy(&attr); if (code != 0) { mTrace("failed to create telemetry thread since :%s", strerror(code)); @@ -303,18 +304,19 @@ int32_t mndInitTelem(SMnode *pMnode) { return 0; } -void mndCleanupTelem(SMnode *pMnode) { - if (!tsTelem.enable) return; +void mndCleanupTelem(SMnode* pMnode) { + STelemMgmt* pMgmt = &pMnode->telemMgmt; + if (!pMgmt->enable) return; - if (taosCheckPthreadValid(tsTelem.thread)) { - pthread_mutex_lock(&tsTelem.lock); - tsTelem.exit = 1; - pthread_cond_signal(&tsTelem.cond); - pthread_mutex_unlock(&tsTelem.lock); + if (taosCheckPthreadValid(pMgmt->thread)) { + pthread_mutex_lock(&pMgmt->lock); + pMgmt->exit = 1; + pthread_cond_signal(&pMgmt->cond); + pthread_mutex_unlock(&pMgmt->lock); - pthread_join(tsTelem.thread, NULL); + pthread_join(pMgmt->thread, NULL); } - pthread_mutex_destroy(&tsTelem.lock); - pthread_cond_destroy(&tsTelem.cond); + pthread_mutex_destroy(&pMgmt->lock); + pthread_cond_destroy(&pMgmt->cond); } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index b99de4a019..2fa0838ee8 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -203,22 +203,25 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->sendMsgToDnodeFp = pOption->sendMsgToDnodeFp; pMnode->sendMsgToMnodeFp = pOption->sendMsgToMnodeFp; pMnode->sendRedirectMsgFp = pOption->sendRedirectMsgFp; - pMnode->sver = pOption->sver; - pMnode->statusInterval = pOption->statusInterval; - pMnode->mnodeEqualVnodeNum = pOption->mnodeEqualVnodeNum; - pMnode->shellActivityTimer = pOption->shellActivityTimer; - pMnode->timezone = strdup(pOption->timezone); - pMnode->locale = strdup(pOption->locale); - pMnode->charset = strdup(pOption->charset); + pMnode->cfg.sver = pOption->cfg.sver; + pMnode->cfg.enableTelem = pOption->cfg.enableTelem; + pMnode->cfg.statusInterval = pOption->cfg.statusInterval; + pMnode->cfg.mnodeEqualVnodeNum = pOption->cfg.mnodeEqualVnodeNum; + pMnode->cfg.shellActivityTimer = pOption->cfg.shellActivityTimer; + pMnode->cfg.timezone = strdup(pOption->cfg.timezone); + pMnode->cfg.locale = strdup(pOption->cfg.locale); + pMnode->cfg.charset = strdup(pOption->cfg.charset); + pMnode->cfg.gitinfo = strdup(pOption->cfg.gitinfo); + pMnode->cfg.buildinfo = strdup(pOption->cfg.buildinfo); if (pMnode->sendMsgToDnodeFp == NULL || pMnode->sendMsgToMnodeFp == NULL || pMnode->sendRedirectMsgFp == NULL || pMnode->putMsgToApplyMsgFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0 || - pMnode->statusInterval < 1 || pOption->mnodeEqualVnodeNum < 0) { + pMnode->cfg.statusInterval < 1 || pOption->cfg.mnodeEqualVnodeNum < 0) { terrno = TSDB_CODE_MND_INVALID_OPTIONS; return -1; } - if (pMnode->timezone == NULL || pMnode->locale == NULL || pMnode->charset == NULL) { + if (pMnode->cfg.timezone == NULL || pMnode->cfg.locale == NULL || pMnode->cfg.charset == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -289,9 +292,11 @@ void mndClose(SMnode *pMnode) { mDebug("start to close mnode"); mndCleanupSteps(pMnode, -1); tfree(pMnode->path); - tfree(pMnode->charset); - tfree(pMnode->locale); - tfree(pMnode->timezone); + tfree(pMnode->cfg.charset); + tfree(pMnode->cfg.locale); + tfree(pMnode->cfg.timezone); + tfree(pMnode->cfg.gitinfo); + tfree(pMnode->cfg.buildinfo); tfree(pMnode); mDebug("mnode is closed"); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 11b40f61f6..3a36ad9b42 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -86,6 +86,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, "Invalid Ref ID") TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, "Ref is already there") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, "Ref is not there") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VERSION_NUMBER, "Invalid version number") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VERSION_STRING, "Invalid version string") +TAOS_DEFINE_ERROR(TSDB_CODE_VERSION_NOT_COMPATIBLE, "Version not compatible") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") diff --git a/source/util/src/tversion.c b/source/util/src/tversion.c new file mode 100644 index 0000000000..3944bd5132 --- /dev/null +++ b/source/util/src/tversion.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "tdef.h" +#include "ulog.h" + +int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { + if (vstr == NULL) { + terrno = TSDB_CODE_INVALID_VERSION_STRING; + return -1; + } + + int32_t vnum[4] = {0}; + int32_t len = strlen(vstr); + char tmp[16] = {0}; + + for (int32_t spos = 0, tpos = 0, vpos = 0; spos < len && vpos < 4; ++spos) { + if (vstr[spos] != '.') { + tmp[spos - tpos] = vstr[spos]; + } else { + vnum[vpos] = atoi(tmp); + memset(tmp, 0, sizeof(tmp)); + vpos++; + tpos = spos + 1; + } + } + + if (vnum[0] <= 0) { + terrno = TSDB_CODE_INVALID_VERSION_STRING; + return -1; + } + + *vint = vnum[0] * 1000000 + vnum[1] * 10000 + vnum[2] * 100 + vnum[3]; + return 0; +} + +int32_t taosVersionIntToStr(int32_t vint, char *vstr, int32_t len) { + int32_t s1 = (vint % 100000000) / 1000000; + int32_t s2 = (vint % 1000000) / 10000; + int32_t s3 = (vint % 10000) / 100; + int32_t s4 = vint % 100; + if (s1 <= 0) { + terrno = TSDB_CODE_INVALID_VERSION_NUMBER; + return -1; + } + + snprintf(vstr, len, "%02d.%02d.%02d.%02d", s1, s2, s3, s4); + return 0; +} + +int32_t taosCheckVersionCompatible(int32_t clientVer, int32_t serverVer, int32_t comparedSegments) { + switch (comparedSegments) { + case 4: + break; + case 3: + clientVer %= 100; + serverVer %= 100; + break; + case 2: + clientVer %= 10000; + serverVer %= 10000; + break; + case 1: + clientVer %= 1000000; + serverVer %= 1000000; + break; + default: + terrno = TSDB_CODE_INVALID_VERSION_NUMBER; + return -1; + } + + if (clientVer == serverVer) { + return 0; + } else { + terrno = TSDB_CODE_VERSION_NOT_COMPATIBLE; + return -1; + } +}